diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 146a7c5..d665273 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -386,14 +386,14 @@ jobs: exit 1 # Re-running site.yml with a bumped version must upgrade every etcd - # server. --forks=1 makes Ansible restart them one at a time so the etcd + # server. k3s_server_serial=1 restarts them one at a time so the etcd # quorum is never lost. - name: Modify the k3s_version in inventory for a site.yml upgrade run: | sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml - name: Run site Playbook to upgrade the servers via a version bump - run: ansible-playbook playbooks/site.yml -i tests/ha.yml --forks=1 + run: ansible-playbook playbooks/site.yml -i tests/ha.yml -e k3s_server_serial=1 # Assert the running (kubelet-reported) version, not the on-disk binary: # without a rolling restart the joined servers stay on the old runtime. diff --git a/README.md b/README.md index e901f57..8ea767e 100644 --- a/README.md +++ b/README.md @@ -171,10 +171,10 @@ ansible-playbook k3s.orchestration.upgrade -i inventory.yml ansible-playbook playbooks/upgrade.yml -i inventory.yml ``` -Re-running the `site.yml` playbook after bumping `k3s_version` performs the same upgrade declaratively: it restarts the k3s services so the cluster picks up the new runtime. On a multi-server (HA) cluster, add `--forks=1` so Ansible restarts the servers one at a time and the etcd quorum is never lost: +Re-running the `site.yml` playbook after bumping `k3s_version` performs the same upgrade declaratively: it restarts the k3s services so the cluster picks up the new runtime. On a multi-server (HA) cluster, add `-e k3s_server_serial=1` so the servers restart one at a time and etcd keeps quorum. With embedded etcd, the next server restarts only after the restarted ones report as etcd voters. By default all servers restart together. Set it through extra vars or `import_playbook` vars, not inventory: `serial` is a play keyword, so inventory variables don't apply to it. ```bash -ansible-playbook playbooks/site.yml -i inventory.yml --forks=1 +ansible-playbook playbooks/site.yml -i inventory.yml -e k3s_server_serial=1 ``` The dedicated `upgrade.yml` playbook remains available and unchanged. diff --git a/playbooks/site.yml b/playbooks/site.yml index d2e0981..2926c71 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -12,6 +12,7 @@ hosts: server gather_facts: false become: true + serial: "{{ k3s_server_serial | default('100%') }}" roles: - role: k3s_server diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index aedeefb..4818d00 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -10,6 +10,11 @@ ansible.builtin.set_fact: k3s_server_installed_version: "{{ k3s_server_version_output.stdout_lines[0].split(' ')[2] }}" +# Limit join checks to completed and current batches during serial installation. +- name: Count the servers this play has already started + ansible.builtin.set_fact: + k3s_server_started_count: "{{ ansible_play_hosts_all.index(ansible_play_batch[-1]) + 1 }}" + - name: Construct K3s server config block: # Start with an empty config @@ -299,8 +304,7 @@ # Always restart a joined server so a re-run of site.yml — whether it only # bumped k3s_version or changed config — brings it onto the new runtime, the # same way the first server above always restarts. On a multi-server cluster - # run the playbook with --forks=1 so the servers restart one at a time and - # the etcd quorum is preserved (see README). + # set k3s_server_serial to 1 so the servers restart one at a time (see README). - name: Restart K3s service when: - ansible_facts.services['k3s.service'] is defined @@ -316,13 +320,15 @@ state: started enabled: true - - name: Verify that all server nodes joined - when: not ansible_check_mode and (groups[server_group] | length) > 1 + - name: Verify that the servers started so far joined + when: not ansible_check_mode ansible.builtin.command: cmd: > k3s kubectl get nodes -l "node-role.kubernetes.io/control-plane=true" -o=jsonpath="{.items[*].metadata.name}" register: k3s_server_nodes - until: k3s_server_nodes.rc == 0 and (k3s_server_nodes.stdout.split() | length) == (groups[server_group] | length) + until: + - k3s_server_nodes.rc == 0 + - (k3s_server_nodes.stdout.split() | length) >= (k3s_server_started_count | int) retries: 20 delay: 10 changed_when: false @@ -335,6 +341,24 @@ run_once: true changed_when: false +- name: Wait for the servers started so far to be etcd voters + when: + - not ansible_check_mode + - (groups[server_group] | length) > 1 + - not use_external_database + ansible.builtin.command: + cmd: >- + k3s kubectl --request-timeout=10s get nodes -l "node-role.kubernetes.io/etcd=true" + -o=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[?(@.type=="EtcdIsVoter")].status}{"\n"}{end}' + register: k3s_server_etcd_voters + until: + - k3s_server_etcd_voters.rc == 0 + - (k3s_server_etcd_voters.stdout.split() | length) >= (k3s_server_started_count | int) + - (k3s_server_etcd_voters.stdout.split() | reject('search', '=True$') | list | length) == 0 + retries: 30 + delay: 10 + changed_when: false + - name: Setup kubectl for user when: user_kubectl block: