diff --git a/README.md b/README.md index ff3f1e7..cdb24fd 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`. -This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.chipzoller.dev/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`. +This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`. If you want more context on how this works, see: diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index eac25c6..6230fb1 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -81,3 +81,49 @@ proxmox_lxc_ct_ids: - 202 - 203 - 204 + +# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache +# (harbor / nexus / docker's official registry / etc). +# Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub), +# or air-gapped environments where your nodes don't have internet access after the initial setup +# (which is still needed for downloading the k3s binary and such). +# k3s's documentation about private registries here: https://docs.k3s.io/installation/private-registry +custom_registries: false +# The registries can be authenticated or anonymous, depending on your registry server configuration. +# If they allow anonymous access, simply remove the following bit from custom_registries_yaml +# configs: +# "registry.domain.com": +# auth: +# username: yourusername +# password: yourpassword +# The following is an example that pulls all images used in this playbook through your private registries. +# It also allows you to pull your own images from your private registry, without having to use imagePullSecrets +# in your deployments. +# If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images, +# you can just remove those from the mirrors: section. +custom_registries_yaml: | + mirrors: + docker.io: + endpoint: + - "https://registry.domain.com/v2/dockerhub" + quay.io: + endpoint: + - "https://registry.domain.com/v2/quayio" + ghcr.io: + endpoint: + - "https://registry.domain.com/v2/ghcrio" + registry.domain.com: + endpoint: + - "https://registry.domain.com" + + configs: + "registry.domain.com": + auth: + username: yourusername + password: yourpassword + +# Only enable and configure these if you access the internet through a proxy +# proxy_env: +# HTTP_PROXY: "http://proxy.domain.local:3128" +# HTTPS_PROXY: "http://proxy.domain.local:3128" +# NO_PROXY: "*.domain.local,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" diff --git a/requirements.txt b/requirements.txt index 0604a64..30cb1d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible-compat==3.0.1 # via molecule -ansible-core==2.15.2 +ansible-core==2.15.4 # via # -r requirements.in # ansible-compat @@ -86,7 +86,7 @@ molecule==4.0.4 # molecule-vagrant molecule-vagrant==1.0.0 # via -r requirements.in -netaddr==0.8.0 +netaddr==0.9.0 # via -r requirements.in nodeenv==1.7.0 # via pre-commit @@ -103,7 +103,7 @@ pluggy==1.0.0 # via molecule pre-commit==2.21.0 # via -r requirements.in -pre-commit-hooks==4.4.0 +pre-commit-hooks==4.5.0 # via -r requirements.in pyasn1==0.4.8 # via diff --git a/roles/k3s_agent/tasks/http_proxy.yml b/roles/k3s_agent/tasks/http_proxy.yml new file mode 100644 index 0000000..f0a68f6 --- /dev/null +++ b/roles/k3s_agent/tasks/http_proxy.yml @@ -0,0 +1,18 @@ +--- + +- name: Create k3s.service.d directory + file: + path: '{{ systemd_dir }}/k3s.service.d' + state: directory + owner: root + group: root + mode: '0755' + + +- name: Copy K3s http_proxy conf file + template: + src: "http_proxy.conf.j2" + dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf" + owner: root + group: root + mode: '0755' diff --git a/roles/k3s_agent/tasks/main.yml b/roles/k3s_agent/tasks/main.yml index 0ce8e08..395c1ac 100644 --- a/roles/k3s_agent/tasks/main.yml +++ b/roles/k3s_agent/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Deploy K3s http_proxy conf + include_tasks: http_proxy.yml + when: proxy_env is defined + - name: Copy K3s service file template: src: "k3s.service.j2" diff --git a/roles/k3s_agent/templates/http_proxy.conf.j2 b/roles/k3s_agent/templates/http_proxy.conf.j2 new file mode 100644 index 0000000..6591d45 --- /dev/null +++ b/roles/k3s_agent/templates/http_proxy.conf.j2 @@ -0,0 +1,4 @@ +[Service] +Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }} +Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }} +Environment=NO_PROXY={{ proxy_env.NO_PROXY }} diff --git a/roles/k3s_custom_registries/defaults/main.yml b/roles/k3s_custom_registries/defaults/main.yml new file mode 100644 index 0000000..704aec7 --- /dev/null +++ b/roles/k3s_custom_registries/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Indicates whether custom registries for k3s should be configured +# Possible values: +# - present +# - absent +state: present diff --git a/roles/k3s_custom_registries/tasks/main.yml b/roles/k3s_custom_registries/tasks/main.yml new file mode 100644 index 0000000..dfe48c2 --- /dev/null +++ b/roles/k3s_custom_registries/tasks/main.yml @@ -0,0 +1,17 @@ +--- + +- name: Create directory /etc/rancher/k3s + file: + path: "/etc/{{ item }}" + state: directory + mode: '0755' + loop: + - rancher + - rancher/k3s + +- name: Insert registries into /etc/rancher/k3s/registries.yaml + blockinfile: + path: /etc/rancher/k3s/registries.yaml + block: "{{ custom_registries_yaml }}" + mode: '0600' + create: true diff --git a/roles/k3s_server/tasks/http_proxy.yml b/roles/k3s_server/tasks/http_proxy.yml new file mode 100644 index 0000000..f0a68f6 --- /dev/null +++ b/roles/k3s_server/tasks/http_proxy.yml @@ -0,0 +1,18 @@ +--- + +- name: Create k3s.service.d directory + file: + path: '{{ systemd_dir }}/k3s.service.d' + state: directory + owner: root + group: root + mode: '0755' + + +- name: Copy K3s http_proxy conf file + template: + src: "http_proxy.conf.j2" + dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf" + owner: root + group: root + mode: '0755' diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 0a8c4b5..030dc22 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -12,11 +12,16 @@ failed_when: false changed_when: false +- name: Deploy K3s http_proxy conf + include_tasks: http_proxy.yml + when: proxy_env is defined + - name: Deploy vip manifest include_tasks: vip.yml - name: Deploy metallb manifest include_tasks: metallb.yml + tags: metallb - name: Init cluster inside the transient k3s-init service command: diff --git a/roles/k3s_server/templates/http_proxy.conf.j2 b/roles/k3s_server/templates/http_proxy.conf.j2 new file mode 100644 index 0000000..6591d45 --- /dev/null +++ b/roles/k3s_server/templates/http_proxy.conf.j2 @@ -0,0 +1,4 @@ +[Service] +Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }} +Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }} +Environment=NO_PROXY={{ proxy_env.NO_PROXY }} diff --git a/roles/k3s_server_post/tasks/main.yml b/roles/k3s_server_post/tasks/main.yml index 84a79db..f88dc08 100644 --- a/roles/k3s_server_post/tasks/main.yml +++ b/roles/k3s_server_post/tasks/main.yml @@ -1,6 +1,7 @@ --- - name: Deploy metallb pool include_tasks: metallb.yml + tags: metallb - name: Remove tmp directory used for manifests file: diff --git a/roles/lxc/handlers/main.yml b/roles/lxc/handlers/main.yml index 7d73985..6450e4c 100644 --- a/roles/lxc/handlers/main.yml +++ b/roles/lxc/handlers/main.yml @@ -2,3 +2,4 @@ - name: Reboot server become: true reboot: + listen: reboot server diff --git a/roles/prereq/defaults/main.yml b/roles/prereq/defaults/main.yml new file mode 100644 index 0000000..e469b0b --- /dev/null +++ b/roles/prereq/defaults/main.yml @@ -0,0 +1,4 @@ +--- +secure_path: + RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' + Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin' diff --git a/roles/prereq/tasks/main.yml b/roles/prereq/tasks/main.yml index b85ae0d..2fffe06 100644 --- a/roles/prereq/tasks/main.yml +++ b/roles/prereq/tasks/main.yml @@ -15,6 +15,7 @@ value: "1" state: present reload: yes + tags: sysctl - name: Enable IPv6 forwarding ansible.posix.sysctl: @@ -22,6 +23,7 @@ value: "1" state: present reload: yes + tags: sysctl - name: Enable IPv6 router advertisements ansible.posix.sysctl: @@ -29,6 +31,7 @@ value: "2" state: present reload: yes + tags: sysctl - name: Add br_netfilter to /etc/modules-load.d/ copy: @@ -53,13 +56,14 @@ loop: - net.bridge.bridge-nf-call-iptables - net.bridge.bridge-nf-call-ip6tables + tags: sysctl - name: Add /usr/local/bin to sudo secure_path lineinfile: - line: 'Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' + line: 'Defaults secure_path = {{ secure_path[ansible_os_family] }}' regexp: "Defaults(\\s)*secure_path(\\s)*=" state: present insertafter: EOF path: /etc/sudoers validate: 'visudo -cf %s' - when: ansible_os_family == "RedHat" + when: ansible_os_family in [ "RedHat", "Suse" ] diff --git a/roles/proxmox_lxc/handlers/main.yml b/roles/proxmox_lxc/handlers/main.yml index 565c882..0d5d983 100644 --- a/roles/proxmox_lxc/handlers/main.yml +++ b/roles/proxmox_lxc/handlers/main.yml @@ -5,7 +5,9 @@ set_fact: proxmox_lxc_filtered_ids: >- {{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }} + listen: reboot containers - name: Reboot container command: "pct reboot {{ item }}" loop: "{{ proxmox_lxc_filtered_ids }}" changed_when: true + listen: reboot containers diff --git a/roles/raspberrypi/handlers/main.yml b/roles/raspberrypi/handlers/main.yml index ac385a7..93e7459 100644 --- a/roles/raspberrypi/handlers/main.yml +++ b/roles/raspberrypi/handlers/main.yml @@ -1,3 +1,4 @@ --- - name: Reboot reboot: + listen: reboot diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index cd3bf72..d75c9ff 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -46,6 +46,15 @@ - /var/lib/rancher/ - /var/lib/cni/ +- name: Remove K3s http_proxy files + file: + name: "{{ item }}" + state: absent + with_items: + - "{{ systemd_dir }}/k3s.service.d" + - "{{ systemd_dir }}/k3s-node.service.d" + when: proxy_env is defined + - name: Reload daemon_reload systemd: daemon_reload: yes diff --git a/site.yml b/site.yml index 8f24982..6dde6b1 100644 --- a/site.yml +++ b/site.yml @@ -3,6 +3,7 @@ hosts: proxmox gather_facts: true become: yes + environment: "{{ proxy_env | default({}) }}" roles: - role: proxmox_lxc when: proxmox_lxc_configure @@ -10,6 +11,7 @@ - name: Prepare k3s nodes hosts: k3s_cluster gather_facts: yes + environment: "{{ proxy_env | default({}) }}" roles: - role: lxc become: true @@ -20,21 +22,27 @@ become: true - role: raspberrypi become: true + - role: k3s_custom_registries + become: true + when: custom_registries - name: Setup k3s servers hosts: master + environment: "{{ proxy_env | default({}) }}" roles: - role: k3s_server become: true - name: Setup k3s agents hosts: node + environment: "{{ proxy_env | default({}) }}" roles: - role: k3s_agent become: true - name: Configure k3s cluster hosts: master + environment: "{{ proxy_env | default({}) }}" roles: - role: k3s_server_post become: true