chore(ci): triggering build

* Add option for install behind http_proxy

* Tidy up http_proxy usage

.github/workflows/ci.yml

@@ -10,6 +10,12 @@ on:
 jobs:
   lint:
     uses: ./.github/workflows/lint.yml
-  test:
+  test-default:
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/test-default.yml
     needs: [lint]
+  test-ipv6:
+    uses: ./.github/workflows/test-ipv6.yml
+    needs: [lint, test-default]
+  test-single-node:
+    uses: ./.github/workflows/test-single-node.yml
+    needs: [lint, test-default, test-ipv6]

.github/workflows/lint.yml

@@ -5,24 +5,24 @@ on:
 jobs:
   pre-commit-ci:
     name: Pre-Commit
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # 4.7.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Cache pip
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # 3.3.2
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
@@ -30,7 +30,7 @@ jobs:
             ${{ runner.os }}-pip-
 
       - name: Cache Ansible
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # 3.3.2
         with:
           path: ~/.ansible/collections
           key: ${{ runner.os }}-ansible-${{ hashFiles('collections/requirements.txt') }}
@@ -59,9 +59,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@af2eb3226618e2494e3d9084f515ad6dcf16e229 # 2.0.1
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@f32435541e24cd6a4700a7f52bb2ec59e80603b1 # 2.0.1
         with:
           allowlist: |
             aws-actions/

.github/workflows/test-default.yml

@@ -0,0 +1,80 @@
+---
+name: Molecule Default
+on:
+  workflow_call:
+jobs:
+  molecule:
+    name: Molecule
+    runs-on: macos-13
+    strategy:
+      matrix:
+        scenario:
+          - default
+      fail-fast: false
+    env:
+      PYTHON_VERSION: "3.11"
+    steps:
+      - name: Check out the codebase
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Configure VirtualBox
+        run: |-
+          sudo mkdir -p /etc/vbox
+          cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
+          * 192.168.30.0/24
+          * fdad:bad:ba55::/64
+          EOF
+
+      - name: Cache pip
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # 3.3.2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Download Vagrant boxes for all scenarios
+        # To save some cache space, all scenarios share the same cache key.
+        # On the other hand, this means that the cache contents should be
+        # the same across all scenarios. This step ensures that.
+        run: ./.github/download-boxes.sh
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # 4.7.1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip' # caching pip dependencies
+
+      - name: Install dependencies
+        run: |
+          echo "::group::Upgrade pip"
+          python3 -m pip install --upgrade pip
+          echo "::endgroup::"
+
+          echo "::group::Install Python requirements from requirements.txt"
+          python3 -m pip install -r requirements.txt
+          echo "::endgroup::"
+
+      - name: Test with molecule
+        run: molecule test --scenario-name ${{ matrix.scenario }}
+        timeout-minutes: 90
+        env:
+          ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
+          ANSIBLE_SSH_RETRIES: 4
+          ANSIBLE_TIMEOUT: 60
+          PY_COLORS: 1
+          ANSIBLE_FORCE_COLOR: 1
+
+      - name: Upload log files
+        if: always() # do this even if a step before has failed
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # 3.1.3
+        with:
+          name: logs
+          path: |
+            ${{ runner.temp }}/logs
+
+      - name: Delete old box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force

.github/workflows/test-ipv6.yml

@@ -0,0 +1,80 @@
+---
+name: Molecule IPv6
+on:
+  workflow_call:
+jobs:
+  molecule:
+    name: Molecule
+    runs-on: macos-13
+    strategy:
+      matrix:
+        scenario:
+          - ipv6
+      fail-fast: false
+    env:
+      PYTHON_VERSION: "3.11"
+    steps:
+      - name: Check out the codebase
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Configure VirtualBox
+        run: |-
+          sudo mkdir -p /etc/vbox
+          cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
+          * 192.168.30.0/24
+          * fdad:bad:ba55::/64
+          EOF
+
+      - name: Cache pip
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # 3.3.2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Download Vagrant boxes for all scenarios
+        # To save some cache space, all scenarios share the same cache key.
+        # On the other hand, this means that the cache contents should be
+        # the same across all scenarios. This step ensures that.
+        run: ./.github/download-boxes.sh
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # 4.7.1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip' # caching pip dependencies
+
+      - name: Install dependencies
+        run: |
+          echo "::group::Upgrade pip"
+          python3 -m pip install --upgrade pip
+          echo "::endgroup::"
+
+          echo "::group::Install Python requirements from requirements.txt"
+          python3 -m pip install -r requirements.txt
+          echo "::endgroup::"
+
+      - name: Test with molecule
+        run: molecule test --scenario-name ${{ matrix.scenario }}
+        timeout-minutes: 90
+        env:
+          ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
+          ANSIBLE_SSH_RETRIES: 4
+          ANSIBLE_TIMEOUT: 60
+          PY_COLORS: 1
+          ANSIBLE_FORCE_COLOR: 1
+
+      - name: Upload log files
+        if: always() # do this even if a step before has failed
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # 3.1.3
+        with:
+          name: logs
+          path: |
+            ${{ runner.temp }}/logs
+
+      - name: Delete old box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force

.github/workflows/test-single-node.yml

@@ -1,24 +1,21 @@
 ---
-name: Test
+name: Molecule Single Node
 on:
   workflow_call:
 jobs:
   molecule:
     name: Molecule
-    runs-on: macos-12
+    runs-on: macos-13
     strategy:
       matrix:
         scenario:
-          - default
-          - ipv6
           - single_node
       fail-fast: false
     env:
       PYTHON_VERSION: "3.11"
-
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
@@ -31,22 +28,13 @@ jobs:
           EOF
 
       - name: Cache pip
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # 3.3.2
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
-      - name: Cache Vagrant boxes
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
-        with:
-          path: |
-            ~/.vagrant.d/boxes
-          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
-          restore-keys: |
-            vagrant-boxes
-
       - name: Download Vagrant boxes for all scenarios
         # To save some cache space, all scenarios share the same cache key.
         # On the other hand, this means that the cache contents should be
@@ -54,7 +42,7 @@ jobs:
         run: ./.github/download-boxes.sh
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # 4.7.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -81,7 +69,7 @@ jobs:
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # 3.1.1
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # 3.1.3
         with:
           name: logs
           path: |

README.md

@@ -4,7 +4,7 @@
 
 This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`.
 
-This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.chipzoller.dev/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
+This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
 
 If you want more context on how this works, see:
 

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.27.4+k3s1
+k3s_version: v1.25.12+k3s1
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -81,3 +81,49 @@ proxmox_lxc_ct_ids:
   - 202
   - 203
   - 204
+
+# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache
+# (harbor / nexus / docker's official registry / etc).
+# Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub),
+# or air-gapped environments where your nodes don't have internet access after the initial setup
+# (which is still needed for downloading the k3s binary and such).
+# k3s's documentation about private registries here: https://docs.k3s.io/installation/private-registry
+custom_registries: false
+# The registries can be authenticated or anonymous, depending on your registry server configuration.
+# If they allow anonymous access, simply remove the following bit from custom_registries_yaml
+#   configs:
+#     "registry.domain.com":
+#       auth:
+#         username: yourusername
+#         password: yourpassword
+# The following is an example that pulls all images used in this playbook through your private registries.
+# It also allows you to pull your own images from your private registry, without having to use imagePullSecrets
+# in your deployments.
+# If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images,
+# you can just remove those from the mirrors: section.
+custom_registries_yaml: |
+  mirrors:
+    docker.io:
+      endpoint:
+        - "https://registry.domain.com/v2/dockerhub"
+    quay.io:
+      endpoint:
+        - "https://registry.domain.com/v2/quayio"
+    ghcr.io:
+      endpoint:
+        - "https://registry.domain.com/v2/ghcrio"
+    registry.domain.com:
+      endpoint:
+        - "https://registry.domain.com"
+
+  configs:
+    "registry.domain.com":
+      auth:
+        username: yourusername
+        password: yourpassword
+
+# Only enable and configure these if you access the internet through a proxy
+# proxy_env:
+#   HTTP_PROXY: "http://proxy.domain.local:3128"
+#   HTTPS_PROXY: "http://proxy.domain.local:3128"
+#   NO_PROXY: "*.domain.local,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"

requirements.txt

@@ -6,7 +6,7 @@
 #
 ansible-compat==3.0.1
     # via molecule
-ansible-core==2.15.2
+ansible-core==2.15.4
     # via
     #   -r requirements.in
     #   ansible-compat
@@ -86,7 +86,7 @@ molecule==4.0.4
     #   molecule-vagrant
 molecule-vagrant==1.0.0
     # via -r requirements.in
-netaddr==0.8.0
+netaddr==0.9.0
     # via -r requirements.in
 nodeenv==1.7.0
     # via pre-commit
@@ -103,7 +103,7 @@ pluggy==1.0.0
     # via molecule
 pre-commit==2.21.0
     # via -r requirements.in
-pre-commit-hooks==4.4.0
+pre-commit-hooks==4.5.0
     # via -r requirements.in
 pyasn1==0.4.8
     # via

roles/k3s_agent/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_agent/tasks/main.yml

@@ -1,5 +1,9 @@
 ---
 
+- name: Deploy K3s http_proxy conf
+  include_tasks: http_proxy.yml
+  when: proxy_env is defined
+
 - name: Copy K3s service file
   template:
     src: "k3s.service.j2"

roles/k3s_agent/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_custom_registries/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether custom registries for k3s should be configured
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/k3s_custom_registries/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Create directory /etc/rancher/k3s
+  file:
+    path: "/etc/{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - rancher
+    - rancher/k3s
+
+- name: Insert registries into /etc/rancher/k3s/registries.yaml
+  blockinfile:
+    path: /etc/rancher/k3s/registries.yaml
+    block: "{{ custom_registries_yaml }}"
+    mode: '0600'
+    create: true

roles/k3s_server/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_server/tasks/main.yml

@@ -12,11 +12,16 @@
   failed_when: false
   changed_when: false
 
+- name: Deploy K3s http_proxy conf
+  include_tasks: http_proxy.yml
+  when: proxy_env is defined
+
 - name: Deploy vip manifest
   include_tasks: vip.yml
 
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
+  tags: metallb
 
 - name: Init cluster inside the transient k3s-init service
   command:

roles/k3s_server/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_server_post/tasks/main.yml

@@ -1,6 +1,7 @@
 ---
 - name: Deploy metallb pool
   include_tasks: metallb.yml
+  tags: metallb
 
 - name: Remove tmp directory used for manifests
   file:

roles/lxc/handlers/main.yml

@@ -2,3 +2,4 @@
 - name: Reboot server
   become: true
   reboot:
+  listen: reboot server

roles/prereq/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+secure_path:
+  RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+  Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin'

roles/prereq/tasks/main.yml

@@ -15,6 +15,7 @@
     value: "1"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Enable IPv6 forwarding
   ansible.posix.sysctl:
@@ -22,6 +23,7 @@
     value: "1"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Enable IPv6 router advertisements
   ansible.posix.sysctl:
@@ -29,6 +31,7 @@
     value: "2"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Add br_netfilter to /etc/modules-load.d/
   copy:
@@ -53,13 +56,14 @@
   loop:
     - net.bridge.bridge-nf-call-iptables
     - net.bridge.bridge-nf-call-ip6tables
+  tags: sysctl
 
 - name: Add /usr/local/bin to sudo secure_path
   lineinfile:
-    line: 'Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+    line: 'Defaults    secure_path = {{ secure_path[ansible_os_family] }}'
     regexp: "Defaults(\\s)*secure_path(\\s)*="
     state: present
     insertafter: EOF
     path: /etc/sudoers
     validate: 'visudo -cf %s'
-  when: ansible_os_family == "RedHat"
+  when: ansible_os_family in [ "RedHat", "Suse" ]

roles/proxmox_lxc/handlers/main.yml

@@ -5,7 +5,9 @@
       set_fact:
         proxmox_lxc_filtered_ids: >-
           {{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }}
+      listen: reboot containers
     - name: Reboot container
       command: "pct reboot {{ item }}"
       loop: "{{ proxmox_lxc_filtered_ids }}"
       changed_when: true
+      listen: reboot containers

roles/raspberrypi/handlers/main.yml

@@ -1,3 +1,4 @@
 ---
 - name: Reboot
   reboot:
+  listen: reboot

roles/reset/tasks/main.yml

@@ -46,6 +46,15 @@
     - /var/lib/rancher/
     - /var/lib/cni/
 
+- name: Remove K3s http_proxy files
+  file:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ systemd_dir }}/k3s.service.d"
+    - "{{ systemd_dir }}/k3s-node.service.d"
+  when: proxy_env is defined
+
 - name: Reload daemon_reload
   systemd:
     daemon_reload: yes

site.yml

@@ -3,6 +3,7 @@
   hosts: proxmox
   gather_facts: true
   become: yes
+  environment: "{{ proxy_env | default({}) }}"
   roles:
     - role: proxmox_lxc
       when: proxmox_lxc_configure
@@ -10,6 +11,7 @@
 - name: Prepare k3s nodes
   hosts: k3s_cluster
   gather_facts: yes
+  environment: "{{ proxy_env | default({}) }}"
   roles:
     - role: lxc
       become: true
@@ -20,21 +22,27 @@
       become: true
     - role: raspberrypi
       become: true
+    - role: k3s_custom_registries
+      become: true
+      when: custom_registries
 
 - name: Setup k3s servers
   hosts: master
+  environment: "{{ proxy_env | default({}) }}"
   roles:
     - role: k3s_server
       become: true
 
 - name: Setup k3s agents
   hosts: node
+  environment: "{{ proxy_env | default({}) }}"
   roles:
     - role: k3s_agent
       become: true
 
 - name: Configure k3s cluster
   hosts: master
+  environment: "{{ proxy_env | default({}) }}"
   roles:
     - role: k3s_server_post
       become: true