From db30128468d4e5f4c568b2d019b083b84d6628c3 Mon Sep 17 00:00:00 2001
From: Techno Tim <timothystewart6@gmail.com>
Date: Tue, 4 Aug 2026 03:52:16 -0500
Subject: [PATCH] fix(ci): retry transient DNS failures on remote downloads and
 cilium install (#690)

- Add shared download_retries/download_delay defaults in k3s_server and k3s_server_post roles
- Retry calico CRD and Tigera operator manifest downloads
- Retry Cilium CLI download and cilium install/upgrade command
- Retry kube-vip cloud provider and MetalLB manifest downloads
- The CI runner's resolver intermittently times out on GitHub-hosted domains
---
 roles/k3s_server/defaults/main.yml     |  6 ++++++
 roles/k3s_server/tasks/kube-vip.yml    |  4 ++++
 roles/k3s_server/tasks/metallb.yml     |  4 ++++
 roles/k3s_server_post/tasks/calico.yml |  8 ++++++++
 roles/k3s_server_post/tasks/cilium.yml | 11 +++++++++++
 5 files changed, 33 insertions(+)

diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml
index 5cdfdd8..4a8022d 100644
--- a/roles/k3s_server/defaults/main.yml
+++ b/roles/k3s_server/defaults/main.yml
@@ -23,6 +23,12 @@ metal_lb_controller_tag_version: v0.16.0
 metal_lb_speaker_tag_version: v0.16.0
 metal_lb_type: native
 
+# Shared retry/delay for remote manifest and asset downloads. The CI runner's
+# resolver intermittently times out on GitHub-hosted domains (helm.cilium.io,
+# raw.githubusercontent.com, github.com), so retry transient DNS/network failures.
+download_retries: 5
+download_delay: 10
+
 retry_count: 20
 
 # yamllint disable rule:line-length
diff --git a/roles/k3s_server/tasks/kube-vip.yml b/roles/k3s_server/tasks/kube-vip.yml
index 29d3f03..90c3a46 100644
--- a/roles/k3s_server/tasks/kube-vip.yml
+++ b/roles/k3s_server/tasks/kube-vip.yml
@@ -15,6 +15,10 @@
     owner: root
     group: root
     mode: "0644"
+  register: kube_vip_manifest_download
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
+  until: kube_vip_manifest_download is succeeded
   when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
 
 - name: Copy kubevip configMap manifest to first master
diff --git a/roles/k3s_server/tasks/metallb.yml b/roles/k3s_server/tasks/metallb.yml
index 4dfb72f..5e70a26 100644
--- a/roles/k3s_server/tasks/metallb.yml
+++ b/roles/k3s_server/tasks/metallb.yml
@@ -15,6 +15,10 @@
     owner: root
     group: root
     mode: "0644"
+  register: metallb_manifest_download
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
+  until: metallb_manifest_download is succeeded
   when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
 
 - name: Set image versions in manifest for metallb-{{ metal_lb_type }}
diff --git a/roles/k3s_server_post/tasks/calico.yml b/roles/k3s_server_post/tasks/calico.yml
index 70dac41..0fb70b4 100644
--- a/roles/k3s_server_post/tasks/calico.yml
+++ b/roles/k3s_server_post/tasks/calico.yml
@@ -18,6 +18,10 @@
         owner: root
         group: root
         mode: "0755"
+      register: calico_crd_download
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
+      until: calico_crd_download is succeeded
 
     - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
       ansible.builtin.get_url:
@@ -26,6 +30,10 @@
         owner: root
         group: root
         mode: "0755"
+      register: tigera_operator_download
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
+      until: tigera_operator_download is succeeded
 
     - name: Apply Calico CRD bundle with server-side apply
       ansible.builtin.command: >-
diff --git a/roles/k3s_server_post/tasks/cilium.yml b/roles/k3s_server_post/tasks/cilium.yml
index 9b7f1ea..78fd24e 100644
--- a/roles/k3s_server_post/tasks/cilium.yml
+++ b/roles/k3s_server_post/tasks/cilium.yml
@@ -66,6 +66,10 @@
             - .tar.gz.sha256sum
           vars:
             cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }}
+          register: cilium_cli_download
+          retries: "{{ download_retries }}"
+          delay: "{{ download_delay }}"
+          until: cilium_cli_download is succeeded
 
         - name: Verify the downloaded tarball
           ansible.builtin.shell: |
@@ -182,6 +186,13 @@
         KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
       register: cilium_install_result
       changed_when: cilium_install_result.rc == 0
+      # cilium install/upgrade fetches the Helm chart from helm.cilium.io, which is
+      # fronted by GitHub Pages and can transiently fail DNS resolution through the
+      # host resolver (intermittent "lookup helm.cilium.io ... i/o timeout"). Retry
+      # so a transient name/network failure does not abort the whole converge play.
+      until: cilium_install_result.rc == 0
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
       when: cilium_installed.rc != 0 or cilium_needs_update
 
     - name: Wait for Cilium resources