diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index efdfe47..319c123 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -19,6 +19,7 @@ jobs: env: STARTING_K3S_VERSION: v1.34.3+k3s1 UPGRADE_K3S_VERSION: v1.35.1+k3s1 + SITE_UPGRADE_K3S_VERSION: v1.35.6+k3s1 # K3s requires privileged containers to run inside Docker and access to cgrougs. steps: - name: Set container OS based on service manager @@ -138,6 +139,37 @@ jobs: - name: Verify K3s upgraded on Agent run: docker exec agent-node k3s --version | grep ${UPGRADE_K3S_VERSION} + # Second upgrade, this time re-running site.yml with a bumped version: it + # must upgrade the running runtime on both the server and the agent, not + # just replace the on-disk binary. + - name: Modify the k3s_version in inventory for a site.yml upgrade + run: | + sed -i "s/k3s_version: .*/k3s_version: ${SITE_UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml + + - name: Run site Playbook to upgrade via a version bump + run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml + + # The install script replaces the on-disk binary regardless of a restart, + # so assert the running version the kubelet reports to the API: it only + # advances once the service was actually restarted onto the new build. + # This is what fails if the version-bump restart regresses. + - name: Verify the running K3s version upgraded on Server and Agent + run: | + for node in server-node agent-node; do + for attempt in 1 2 3 4 5 6; do + running=$(docker exec server-node k3s kubectl get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1) + echo "$node running version: $running" + if echo "$running" | grep -q "${SITE_UPGRADE_K3S_VERSION}"; then + break + fi + if [ "$attempt" -eq 6 ]; then + echo "$node did not reach ${SITE_UPGRADE_K3S_VERSION}" + exit 1 + fi + sleep 10 + done + done + - name: Wait for all deployments to be ready run: | for attempt in 1 2 3 4 5 6; do @@ -324,6 +356,51 @@ jobs: done exit 1 + # Re-running site.yml with a bumped version must upgrade every etcd + # server. --forks=1 makes Ansible restart them one at a time so the etcd + # quorum is never lost. + - name: Modify the k3s_version in inventory for a site.yml upgrade + run: | + sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml + + - name: Run site Playbook to upgrade the servers via a version bump + run: ansible-playbook playbooks/site.yml -i tests/ha.yml --forks=1 + + # Assert the running (kubelet-reported) version, not the on-disk binary: + # without a rolling restart the joined servers stay on the old runtime. + - name: Verify the running K3s version upgraded on all servers via site.yml + run: | + docker exec server-node1 k3s kubectl get nodes -o wide + for SERVER in $SERVERS; do + for attempt in 1 2 3 4 5 6; do + running=$(docker exec server-node1 k3s kubectl get node "$SERVER" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1) + echo "$SERVER running version: $running" + if echo "$running" | grep -q v1.34.3; then + break + fi + if [ "$attempt" -eq 6 ]; then + echo "$SERVER did not reach v1.34.3" + exit 1 + fi + sleep 10 + done + done + + - name: Wait for all deployments to be ready after the site.yml upgrade + run: | + for attempt in 1 2 3 4 5 6; do + echo "Attempt $attempt: checking deployments" + output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1) + echo "$output" + if ! echo "$output" | grep -q "" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then + exit 0 + fi + if [ "$attempt" -lt 6 ]; then + sleep 15 + fi + done + exit 1 + - name: Debug nodes/pods on failure if: failure() run: | diff --git a/README.md b/README.md index 4163c86..fb546c4 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,14 @@ ansible-playbook k3s.orchestration.upgrade -i inventory.yml ansible-playbook playbooks/upgrade.yml -i inventory.yml ``` +Re-running the `site.yml` playbook after bumping `k3s_version` performs the same upgrade declaratively: it restarts the k3s services so the cluster picks up the new runtime. On a multi-server (HA) cluster, add `--forks=1` so Ansible restarts the servers one at a time and the etcd quorum is never lost: + +```bash +ansible-playbook playbooks/site.yml -i inventory.yml --forks=1 +``` + +The dedicated `upgrade.yml` playbook remains available and unchanged. + ## Airgap Install Airgap installation is supported via the `airgap_dir` variable. This variable should be set to the path of a directory containing the K3s binary and images. The release artifacts can be downloaded from the [K3s Releases](https://github.com/k3s-io/k3s/releases). You must download the appropriate images for you architecture (any of the compression formats will work). Additionally, you must run the `airgap` role to set up the airgapped environment. diff --git a/roles/k3s_agent/tasks/main.yml b/roles/k3s_agent/tasks/main.yml index a68a7ea..4063fa7 100644 --- a/roles/k3s_agent/tasks/main.yml +++ b/roles/k3s_agent/tasks/main.yml @@ -95,8 +95,12 @@ loop: - "K3S_TOKEN={{ token }}" +# Always restart so a re-run of site.yml — whether it changed the agent config +# or bumped k3s_version — reliably brings the agent onto the new state. The +# install script leaves the service stopped (INSTALL_K3S_SKIP_START), so a plain +# restart also covers the first install. - name: Enable and start K3s agent ansible.builtin.service: name: k3s-agent - state: "{{ 'restarted' if _agent_config_result.changed else 'started' }}" + state: restarted enabled: true diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 77e5c6a..915d93b 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -294,11 +294,15 @@ ansible.builtin.systemd: daemon_reload: true + # Always restart a joined server so a re-run of site.yml — whether it only + # bumped k3s_version or changed config — brings it onto the new runtime, the + # same way the first server above always restarts. On a multi-server cluster + # run the playbook with --forks=1 so the servers restart one at a time and + # the etcd quorum is preserved (see README). - name: Restart K3s service when: - ansible_facts.services['k3s.service'] is defined - ansible_facts.services['k3s.service'].state == 'running' - - k3s_server_config_result.changed ansible.builtin.service: name: k3s state: restarted