From 62b80237ef9fd335f6ff6ff422941d4f3bc88a48 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Sun, 12 May 2024 12:42:51 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9E=95=20Add=20default=20values=20to=20r?= =?UTF-8?q?oles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/k3s_agent/defaults/main.yml | 2 ++ roles/k3s_server/defaults/main.yml | 2 ++ roles/reset/defaults/main.yml | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 roles/k3s_agent/defaults/main.yml create mode 100644 roles/reset/defaults/main.yml diff --git a/roles/k3s_agent/defaults/main.yml b/roles/k3s_agent/defaults/main.yml new file mode 100644 index 0000000..0b45925 --- /dev/null +++ b/roles/k3s_agent/defaults/main.yml @@ -0,0 +1,2 @@ +--- +systemd_dir: /etc/systemd/system diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index 46f1528..1ab5b83 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -18,3 +18,5 @@ server_init_args: >- --token {{ k3s_token }} {% endif %} {{ extra_server_args | default('') }} + +systemd_dir: /etc/systemd/system diff --git a/roles/reset/defaults/main.yml b/roles/reset/defaults/main.yml new file mode 100644 index 0000000..0b45925 --- /dev/null +++ b/roles/reset/defaults/main.yml @@ -0,0 +1,2 @@ +--- +systemd_dir: /etc/systemd/system From 16fb511d49b99d519950053b1167b37c7c3ed6f2 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 21 May 2024 09:23:52 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9A=9A=20Move=20to=20use=20meta=20fil?= =?UTF-8?q?es=20for=20roles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/download/meta/main.yml | 8 +++++++ roles/k3s_agent/defaults/main.yml | 3 +++ roles/k3s_agent/meta/main.yml | 29 +++++++++++++++++++++++ roles/k3s_custom_registries/meta/main.yml | 19 +++++++++++++++ roles/k3s_server/defaults/main.yml | 4 ++-- roles/k3s_server/meta/main.yml | 25 +++++++++++++++++++ 6 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 roles/download/meta/main.yml create mode 100644 roles/k3s_agent/meta/main.yml create mode 100644 roles/k3s_custom_registries/meta/main.yml create mode 100644 roles/k3s_server/meta/main.yml diff --git a/roles/download/meta/main.yml b/roles/download/meta/main.yml new file mode 100644 index 0000000..e7911d5 --- /dev/null +++ b/roles/download/meta/main.yml @@ -0,0 +1,8 @@ +--- +argument_specs: + main: + short_description: Manage the downloading of K3S binaries + options: + k3s_version: + description: The desired version of K3S + required: true diff --git a/roles/k3s_agent/defaults/main.yml b/roles/k3s_agent/defaults/main.yml index 0b45925..037c1f3 100644 --- a/roles/k3s_agent/defaults/main.yml +++ b/roles/k3s_agent/defaults/main.yml @@ -1,2 +1,5 @@ --- +is_pxe_booted: false +extra_agent_args: "" +group_name_master: master systemd_dir: /etc/systemd/system diff --git a/roles/k3s_agent/meta/main.yml b/roles/k3s_agent/meta/main.yml new file mode 100644 index 0000000..a2f694c --- /dev/null +++ b/roles/k3s_agent/meta/main.yml @@ -0,0 +1,29 @@ +--- +argument_specs: + main: + short_description: Setup k3s agents + options: + apiserver_endpoint: + description: Virtual ip-address configured on each master + required: true + + extra_agent_args: + description: Virtual ip-address configured on each master + default: Arguments passed to the systemd service + + group_name_master: + description: Name of the master group + default: master + + proxy_env: + type: dict + description: Internet proxy configurations + default: ~ + options: + HTTP_PROXY: + HTTPS_PROXY: + NO_PROXY: + + systemd_dir: + description: Path to systemd services + default: /etc/systemd/system diff --git a/roles/k3s_custom_registries/meta/main.yml b/roles/k3s_custom_registries/meta/main.yml new file mode 100644 index 0000000..0ec0e7c --- /dev/null +++ b/roles/k3s_custom_registries/meta/main.yml @@ -0,0 +1,19 @@ +--- +argument_specs: + main: + short_description: Configure the use of a custom container registry + options: + custom_registries_yaml: + description: + - YAML block defining custom registries. + - > + The following is an example that pulls all images used in + this playbook through your private registries. + - > + It also allows you to pull your own images from your private + registry, without having to use imagePullSecrets in your + deployments. + - > + If all you need is your own images and you don't care about + caching the docker/quay/ghcr.io images, you can just remove + those from the mirrors: section. diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index 1ab5b83..f088c96 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -2,9 +2,9 @@ # If you want to explicitly define an interface that ALL control nodes # should use to propagate the VIP, define it here. Otherwise, kube-vip # will determine the right interface automatically at runtime. -kube_vip_iface: null +kube_vip_iface: ~ -# Name of the master group +kube_vip_cloud_provider_tag_version: main group_name_master: master # yamllint disable rule:line-length diff --git a/roles/k3s_server/meta/main.yml b/roles/k3s_server/meta/main.yml new file mode 100644 index 0000000..cbe37ea --- /dev/null +++ b/roles/k3s_server/meta/main.yml @@ -0,0 +1,25 @@ +--- +argument_specs: + main: + short_description: Setup k3s servers + options: + group_name_master: + description: Name of the master group + default: master + + kube_vip_cloud_provider_tag_version: + description: Tag for kube-vip-cloud-provider manifest when enabled + default: main + + kube_vip_lb_ip_range: + description: IP range for kube-vip load balancer + default: ~ + + proxy_env: + type: dict + description: Internet proxy configurations + default: ~ + options: + HTTP_PROXY: + HTTPS_PROXY: + NO_PROXY: From aa2738ae83bf60a84a1b0efab8deed5ef8f328de Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Sun, 9 Jun 2024 13:43:38 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=9B=A0=20Fix=20descriptions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/k3s_agent/defaults/main.yml | 1 - roles/k3s_agent/meta/main.yml | 9 +++++++-- roles/k3s_agent/templates/k3s.service.j2 | 2 +- roles/k3s_custom_registries/defaults/main.yml | 6 ------ roles/k3s_custom_registries/meta/main.yml | 1 + 5 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 roles/k3s_custom_registries/defaults/main.yml diff --git a/roles/k3s_agent/defaults/main.yml b/roles/k3s_agent/defaults/main.yml index 037c1f3..bdf76ae 100644 --- a/roles/k3s_agent/defaults/main.yml +++ b/roles/k3s_agent/defaults/main.yml @@ -1,5 +1,4 @@ --- -is_pxe_booted: false extra_agent_args: "" group_name_master: master systemd_dir: /etc/systemd/system diff --git a/roles/k3s_agent/meta/main.yml b/roles/k3s_agent/meta/main.yml index a2f694c..04f856a 100644 --- a/roles/k3s_agent/meta/main.yml +++ b/roles/k3s_agent/meta/main.yml @@ -8,21 +8,26 @@ argument_specs: required: true extra_agent_args: - description: Virtual ip-address configured on each master - default: Arguments passed to the systemd service + description: Extra arguments for agents nodes group_name_master: description: Name of the master group default: master + k3s_token: + description: Token used to communicate between masters + proxy_env: type: dict description: Internet proxy configurations default: ~ options: HTTP_PROXY: + required: true HTTPS_PROXY: + required: true NO_PROXY: + required: true systemd_dir: description: Path to systemd services diff --git a/roles/k3s_agent/templates/k3s.service.j2 b/roles/k3s_agent/templates/k3s.service.j2 index dac88de..52aa272 100644 --- a/roles/k3s_agent/templates/k3s.service.j2 +++ b/roles/k3s_agent/templates/k3s.service.j2 @@ -12,7 +12,7 @@ ExecStart=/usr/local/bin/k3s agent \ --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \ {% if is_pxe_booted | default(false) %}--snapshotter native \ {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \ - {{ extra_agent_args | default("") }} + {{ extra_agent_args }} KillMode=process Delegate=yes LimitNOFILE=1048576 diff --git a/roles/k3s_custom_registries/defaults/main.yml b/roles/k3s_custom_registries/defaults/main.yml deleted file mode 100644 index 704aec7..0000000 --- a/roles/k3s_custom_registries/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# Indicates whether custom registries for k3s should be configured -# Possible values: -# - present -# - absent -state: present diff --git a/roles/k3s_custom_registries/meta/main.yml b/roles/k3s_custom_registries/meta/main.yml index 0ec0e7c..3c0878f 100644 --- a/roles/k3s_custom_registries/meta/main.yml +++ b/roles/k3s_custom_registries/meta/main.yml @@ -17,3 +17,4 @@ argument_specs: If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images, you can just remove those from the mirrors: section. + required: true From a1b108a651ba34cfe79f3f8cddba28807ff7ecf3 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Sun, 9 Jun 2024 13:44:03 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=E2=9E=95=20Add=20meta=20for=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/k3s_server/defaults/main.yml | 18 +++++--- roles/k3s_server/meta/main.yml | 60 +++++++++++++++++++++++++++ roles/k3s_server/tasks/http_proxy.yml | 1 - roles/k3s_server/tasks/main.yml | 1 - 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index f088c96..6ca801a 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -1,12 +1,18 @@ --- -# If you want to explicitly define an interface that ALL control nodes -# should use to propagate the VIP, define it here. Otherwise, kube-vip -# will determine the right interface automatically at runtime. -kube_vip_iface: ~ +extra_server_args: "" -kube_vip_cloud_provider_tag_version: main group_name_master: master +kube_vip_iface: ~ +kube_vip_cloud_provider_tag_version: main +kube_vip_tag_version: v0.7.2 + +metal_lb_controller_tag_version: v0.14.3 +metal_lb_speaker_tag_version: v0.14.3 +metal_lb_type: native + +retry_count: 20 + # yamllint disable rule:line-length server_init_args: >- {% if groups[group_name_master | default('master')] | length > 1 %} @@ -17,6 +23,6 @@ server_init_args: >- {% endif %} --token {{ k3s_token }} {% endif %} - {{ extra_server_args | default('') }} + {{ extra_server_args }} systemd_dir: /etc/systemd/system diff --git a/roles/k3s_server/meta/main.yml b/roles/k3s_server/meta/main.yml index cbe37ea..ea5f151 100644 --- a/roles/k3s_server/meta/main.yml +++ b/roles/k3s_server/meta/main.yml @@ -3,10 +3,41 @@ argument_specs: main: short_description: Setup k3s servers options: + apiserver_endpoint: + description: Virtual ip-address configured on each master + required: true + + cilium_bgp: + description: + - Enable cilium BGP control plane for LB services and pod cidrs. + - Disables the use of MetalLB. + type: bool + default: ~ + + cilium_iface: + description: The network interface used for when Cilium is enabled + default: ~ + + extra_server_args: + description: Extra arguments for server nodes + default: "" + group_name_master: description: Name of the master group default: master + kube_vip_iface: + description: + - Explicitly define an interface that ALL control nodes + - should use to propagate the VIP, define it here. + - Otherwise, kube-vip will determine the right interface + - automatically at runtime. + default: ~ + + kube_vip_tag_version: + description: Image tag for kube-vip + default: v0.7.2 + kube_vip_cloud_provider_tag_version: description: Tag for kube-vip-cloud-provider manifest when enabled default: main @@ -15,11 +46,40 @@ argument_specs: description: IP range for kube-vip load balancer default: ~ + metal_lb_controller_tag_version: + description: Image tag for MetalLB + default: v0.14.3 + + metal_lb_speaker_tag_version: + description: Image tag for MetalLB + default: v0.14.3 + + metal_lb_type: + choices: + - frr + - native + default: native + proxy_env: type: dict description: Internet proxy configurations default: ~ options: HTTP_PROXY: + required: true HTTPS_PROXY: + required: true NO_PROXY: + required: true + + retry_count: + description: Amount of retries when verifying that nodes joined + type: int + default: 20 + + server_init_args: + description: Arguments for server nodes + + systemd_dir: + description: Path to systemd services + default: /etc/systemd/system diff --git a/roles/k3s_server/tasks/http_proxy.yml b/roles/k3s_server/tasks/http_proxy.yml index f0a68f6..7161c3a 100644 --- a/roles/k3s_server/tasks/http_proxy.yml +++ b/roles/k3s_server/tasks/http_proxy.yml @@ -1,5 +1,4 @@ --- - - name: Create k3s.service.d directory file: path: '{{ systemd_dir }}/k3s.service.d' diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index ae2d8e9..fa74ccd 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Stop k3s-init systemd: name: k3s-init From 39361d2d8e3fd1ff58a5d2aafa17a0d1c41cf41c Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Sun, 9 Jun 2024 14:11:14 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/k3s_server_post/defaults/main.yml | 20 +++++- roles/k3s_server_post/meta/main.yml | 94 +++++++++++++++++++++++++ roles/k3s_server_post/tasks/cilium.yml | 10 +-- 3 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 roles/k3s_server_post/meta/main.yml diff --git a/roles/k3s_server_post/defaults/main.yml b/roles/k3s_server_post/defaults/main.yml index bbf9629..982f605 100644 --- a/roles/k3s_server_post/defaults/main.yml +++ b/roles/k3s_server_post/defaults/main.yml @@ -1,6 +1,20 @@ --- -# Timeout to wait for MetalLB services to come up -metal_lb_available_timeout: 240s +bpf_lb_algorithm: maglev +bpf_lb_mode: hybrid -# Name of the master group +calico_ebpf: false +calico_tag: v3.27.2 + +cilium_bgp: false +cilium_exportPodCIDR: true +cilium_hubble: true +cilium_mode: native + +cluster_cidr: 10.52.0.0/16 +enable_bpf_masquerade: true +kube_proxy_replacement: true group_name_master: master + +metal_lb_mode: layer2 +metal_lb_available_timeout: 240s +metal_lb_controller_tag_version: v0.14.3 diff --git a/roles/k3s_server_post/meta/main.yml b/roles/k3s_server_post/meta/main.yml new file mode 100644 index 0000000..67dd5db --- /dev/null +++ b/roles/k3s_server_post/meta/main.yml @@ -0,0 +1,94 @@ +--- +argument_specs: + main: + short_description: Configure k3s cluster + options: + apiserver_endpoint: + description: Virtual ip-address configured on each master + required: true + + bpf_lb_algorithm: + description: BPF lb algorithm + default: maglev + + bpf_lb_mode: + description: BPF lb mode + default: hybrid + + calico_ebpf: + description: Use eBPF dataplane instead of iptables + type: bool + default: false + + calico_iface: + description: The network interface used for when Calico is enabled + default: ~ + + calico_tag: + description: Calico version tag + default: v3.27.2 + + cilium_bgp: + description: + - Enable cilium BGP control plane for LB services and pod cidrs. + - Disables the use of MetalLB. + type: bool + default: false + + cilium_exportPodCIDR: + description: Export pod CIDR + type: bool + default: true + + cilium_hubble: + description: Enable Cilium Hubble + type: bool + default: true + + cilium_iface: + description: The network interface used for when Cilium is enabled + default: ~ + + cilium_mode: + description: Inner-node communication mode + default: native + choices: + - native + - routed + + cluster_cidr: + description: Inner-cluster IP range + default: 10.52.0.0/16 + + enable_bpf_masquerade: + description: Use IP masquerading + type: bool + default: true + + group_name_master: + description: Name of the master group + default: master + + kube_proxy_replacement: + description: Replace the native kube-proxy with Cilium + type: bool + default: true + + kube_vip_lb_ip_range: + description: IP range for kube-vip load balancer + default: ~ + + metal_lb_available_timeout: + description: Wait for MetalLB resources + default: 240s + + metal_lb_controller_tag_version: + description: Image tag for MetalLB + default: v0.14.3 + + metal_lb_mode: + description: Metallb mode + default: layer2 + choices: + - bgp + - layer2 diff --git a/roles/k3s_server_post/tasks/cilium.yml b/roles/k3s_server_post/tasks/cilium.yml index 41c181c..93876e9 100644 --- a/roles/k3s_server_post/tasks/cilium.yml +++ b/roles/k3s_server_post/tasks/cilium.yml @@ -172,17 +172,17 @@ {% endif %} --helm-set k8sServiceHost="127.0.0.1" --helm-set k8sServicePort="6444" - --helm-set routingMode={{ cilium_mode | default("native") }} + --helm-set routingMode={{ cilium_mode }} --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }} - --helm-set kubeProxyReplacement={{ kube_proxy_replacement | default("true") }} - --helm-set bpf.masquerade={{ enable_bpf_masquerade | default("true") }} + --helm-set kubeProxyReplacement={{ kube_proxy_replacement }} + --helm-set bpf.masquerade={{ enable_bpf_masquerade }} --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }} --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }} {% if kube_proxy_replacement is not false %} - --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm | default("maglev") }} - --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode | default("hybrid") }} + --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }} + --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }} {% endif %} environment: KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"