From 4c50fbbe103991b5ab43961a725debdbee0cc5d7 Mon Sep 17 00:00:00 2001 From: Timothy Stewart Date: Sat, 1 Aug 2026 01:25:14 -0500 Subject: [PATCH] fix(k3s-server): stabilize single-server bootstrap - delegate cgroups for transient K3s server units\n- verify inventory node registration without legacy role labels\n- wait for bootstrap CRDs before replacing the transient service --- .github/scripts/test-k3s-server-bootstrap.sh | 35 ++++++++++++++++++++ .pre-commit-config.yaml | 6 ++++ roles/k3s_server/tasks/join_master.yml | 5 ++- roles/k3s_server/tasks/main.yml | 35 +++++++++++++++++--- 4 files changed, 76 insertions(+), 5 deletions(-) create mode 100755 .github/scripts/test-k3s-server-bootstrap.sh diff --git a/.github/scripts/test-k3s-server-bootstrap.sh b/.github/scripts/test-k3s-server-bootstrap.sh new file mode 100755 index 0000000..c041112 --- /dev/null +++ b/.github/scripts/test-k3s-server-bootstrap.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +main_tasks="$repo_root/roles/k3s_server/tasks/main.yml" +join_tasks="$repo_root/roles/k3s_server/tasks/join_master.yml" + +for task_file in "$main_tasks" "$join_tasks"; do + for property in \ + 'Delegate=yes' \ + 'TasksMax=infinity' \ + 'KillMode=process' \ + 'LimitNOFILE=1048576' \ + 'LimitNPROC=infinity' \ + 'LimitCORE=infinity'; do + grep -Fq -- "$property" "$task_file" || { + printf '%s is missing transient K3s property %s\n' "$task_file" "$property" >&2 + exit 1 + } + done +done + +if grep -Fq -- "node-role.kubernetes.io/master=true' -o=jsonpath" "$main_tasks"; then + printf 'control-plane registration still depends on the optional legacy master role key\n' >&2 + exit 1 +fi + +grep -Fq -- "map('extract', hostvars, 'ansible_hostname')" "$main_tasks" +grep -Fq -- 'difference(nodes.stdout.split())' "$main_tasks" +grep -Fq -- 'crd/addons.k3s.cattle.io' "$main_tasks" +grep -Fq -- 'crd/helmcharts.helm.cattle.io' "$main_tasks" +grep -Fq -- 'crd/helmchartconfigs.helm.cattle.io' "$main_tasks" + +printf 'K3s transient bootstrap regression test passed\n' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8528df..b3433e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,3 +56,9 @@ repos: language: system pass_filenames: false files: ^\.github/scripts/(prepare-vagrant-box-masters|test-prepare-vagrant-box-masters)\.sh$ + - id: k3s-server-bootstrap-test + name: K3s transient bootstrap test + entry: .github/scripts/test-k3s-server-bootstrap.sh + language: system + pass_filenames: false + files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$ diff --git a/roles/k3s_server/tasks/join_master.yml b/roles/k3s_server/tasks/join_master.yml index ca327ef..59ae78a 100644 --- a/roles/k3s_server/tasks/join_master.yml +++ b/roles/k3s_server/tasks/join_master.yml @@ -12,7 +12,10 @@ - name: Join transient k3s-init service for {{ joining_master }} ansible.builtin.command: cmd: >- - systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init + systemd-run -p RestartSec=2 -p Restart=on-failure + -p Delegate=yes -p TasksMax=infinity -p KillMode=process + -p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity + --unit=k3s-init k3s server {{ hostvars[joining_master].k3s_server_init_args }} creates: "{{ systemd_dir }}/k3s-init.service" delegate_to: "{{ joining_master }}" diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 79482cb..d0306c9 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -51,6 +51,8 @@ ansible.builtin.command: cmd: >- systemd-run -p RestartSec=2 -p Restart=on-failure + -p Delegate=yes -p TasksMax=infinity -p KillMode=process + -p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity --unit=k3s-init k3s server {{ server_init_args }} creates: "{{ systemd_dir }}/k3s-init.service" no_log: true @@ -80,13 +82,38 @@ loop_var: joining_master when: groups[group_name_master | default('master')] | length > 1 - - name: Verify that all nodes actually joined (check k3s-init.service if this fails) + - name: Verify that all expected control-plane nodes registered ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length + cmd: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes + -o=jsonpath='{.items[*].metadata.name}' register: nodes - until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length + until: + - nodes.rc == 0 + - expected_control_plane_names | difference(nodes.stdout.split()) | length == 0 retries: "{{ retry_count | default(20) }}" - delay: 10 + delay: 2 + changed_when: false + vars: + expected_control_plane_names: >- + {{ + groups[group_name_master | default('master')] + | map('extract', hostvars, 'ansible_hostname') + | list + }} + + - name: Wait for K3s bootstrap CRDs to become established + ansible.builtin.command: + cmd: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} wait + --for=condition=Established --timeout=5s + crd/addons.k3s.cattle.io + crd/helmcharts.helm.cattle.io + crd/helmchartconfigs.helm.cattle.io + register: bootstrap_crds + until: bootstrap_crds.rc == 0 + retries: "{{ retry_count | default(20) }}" + delay: 2 changed_when: false always: - name: Save logs of k3s-init.service