Merge pull request 'master' (#1) from tim/k3s-ansible:master into master

Reviewed-on: #1

.ansible-lint

@@ -1,3 +1,20 @@
 ---
+exclude_paths:
+  # default paths
+  - '.cache/'
+  - '.github/'
+  - 'test/fixtures/formatting-before/'
+  - 'test/fixtures/formatting-prettier/'
+
+  # The "converge" and "reset" playbooks use import_playbook in
+  # conjunction with the "env" lookup plugin, which lets the
+  # syntax check of ansible-lint fail.
+  - 'molecule/**/converge.yml'
+  - 'molecule/**/prepare.yml'
+  - 'molecule/**/reset.yml'
+
+  # The file was generated by galaxy ansible - don't mess with it.
+  - 'galaxy.yml'
+
 skip_list:
   - 'fqcn-builtins'

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+[*]
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf
+max_line_length = off
+[Makefile]
+indent_style = tab
+[*.go]
+indent_style = tab

.github/ISSUE_TEMPLATE.md

@@ -37,6 +37,11 @@ systemd_dir: ""
 
 flannel_iface: ""
 
+#calico_iface: ""
+calico_ebpf: ""
+calico_cidr: ""
+calico_tag: ""
+
 apiserver_endpoint: ""
 
 k3s_token: "NA"
@@ -46,6 +51,9 @@ extra_agent_args: ""
 
 kube_vip_tag_version: ""
 
+kube_vip_cloud_provider_tag_version: ""
+kube_vip_lb_ip_range: ""
+
 metal_lb_speaker_tag_version: ""
 metal_lb_controller_tag_version: ""
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,4 +11,5 @@
 - [ ] Ran `site.yml` playbook
 - [ ] Ran `reset.yml` playbook
 - [ ] Did not add any unnecessary changes
+- [ ] Ran pre-commit install at least once before committing
 - [ ] 🚀

.github/dependabot.yml

@@ -9,3 +9,18 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    rebase-strategy: "auto"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    rebase-strategy: "auto"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/download-boxes.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# download-boxes.sh
+# Check all molecule.yml files for required Vagrant boxes and download the ones that are not
+# already present on the system.
+
+set -euo pipefail
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+PROVIDER=virtualbox
+
+# Read all boxes for all platforms from the "molecule.yml" files
+all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
+    yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"
+    grep --invert-match --regexp=--- | # Filter out file separators
+    sort |
+    uniq)
+
+# Read the boxes that are currently present on the system (for the current provider)
+present_boxes=$(
+    (vagrant box list |
+        grep "${PROVIDER}" | # Filter by boxes available for the current provider
+        awk '{print $1;}' |  # The box name is the first word in each line
+        sort |
+        uniq) ||
+        echo "" # In case any of these commands fails, just use an empty list
+)
+
+# The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
+download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
+
+# Actually download the necessary boxes
+if [ -n "${download_boxes}" ]; then
+    echo "${download_boxes}" | while IFS= read -r box; do
+        vagrant box add --provider "${PROVIDER}" "${box}"
+    done
+fi

.github/workflows/cache.yml

@@ -0,0 +1,42 @@
+---
+name: "Cache"
+on:
+  workflow_call:
+jobs:
+  molecule:
+    name: cache
+    runs-on: self-hosted
+    env:
+      PYTHON_VERSION: "3.11"
+
+    steps:
+      - name: Check out the codebase
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip' # caching pip dependencies
+
+      - name: Cache Vagrant boxes
+        id: cache-vagrant
+        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        with:
+          lookup-only: true #if it exists, we don't need to restore and can skip the next step
+          path: |
+            ~/.vagrant.d/boxes
+          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+          restore-keys: |
+            vagrant-boxes
+
+      - name: Download Vagrant boxes for all scenarios
+        # To save some cache space, all scenarios share the same cache key.
+        # On the other hand, this means that the cache contents should be
+        # the same across all scenarios. This step ensures that.
+        if: steps.cache-vagrant.outputs.cache-hit != 'true' # only run if false since this is just a cache step
+        run: |
+          ./.github/download-boxes.sh
+          vagrant box list

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+---
+name: "CI"
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    paths-ignore:
+      - '**/.gitignore'
+      - '**/FUNDING.yml'
+      - '**/host.ini'
+      - '**/*.md'
+      - '**/.editorconfig'
+      - '**/ansible.example.cfg'
+      - '**/deploy.sh'
+      - '**/LICENSE'
+      - '**/reboot.sh'
+      - '**/reset.sh'
+jobs:
+  pre:
+    uses: ./.github/workflows/cache.yml
+  lint:
+    uses: ./.github/workflows/lint.yml
+    needs: [pre]
+  test:
+    uses: ./.github/workflows/test.yml
+    needs: [pre, lint]

.github/workflows/lint.yml

@@ -1,30 +1,54 @@
 ---
 name: Linting
 on:
-  pull_request:
+  workflow_call:
-  push:
-    branches:
-      - master
-
 jobs:
-  ansible-lint:
+  pre-commit-ci:
-    name: YAML Lint + Ansible Lint
+    name: Pre-Commit
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    env:
+      PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
-
-      - name: Set up Python 3.x
-        uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 #4.0.2
         with:
-          python-version: '3.x'
+          ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Install test dependencies
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        run: pip3 install yamllint ansible-lint ansible
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip' # caching pip dependencies
 
-      - name: Run yamllint
+      - name: Restore Ansible cache
-        run: yamllint .
+        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        with:
+          path: ~/.ansible/collections
+          key: ansible-${{ hashFiles('collections/requirements.yml') }}
 
-      - name: Run ansible-lint
+      - name: Install dependencies
-        run: ansible-lint
+        run: |
+          echo "::group::Upgrade pip"
+          python3 -m pip install --upgrade pip
+          echo "::endgroup::"
+
+          echo "::group::Install Python requirements from requirements.txt"
+          python3 -m pip install -r requirements.txt
+          echo "::endgroup::"
+
+      - name: Run pre-commit
+        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # 3.0.0
+
+  ensure-pinned-actions:
+    name: Ensure SHA Pinned Actions
+    runs-on: self-hosted
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
+      - name: Ensure SHA pinned actions
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@ba37328d4ea95eaf8b3bd6c6cef308f709a5f2ec # 3.0.3
+        with:
+          allowlist: |
+            aws-actions/
+            docker/login-action

.github/workflows/test.yml

@@ -1,69 +1,125 @@
 ---
 name: Test
 on:
-  pull_request:
+  workflow_call:
-  push:
-    branches:
-      - master
-
 jobs:
-  vagrant:
+  molecule:
-    name: Vagrant
+    name: Molecule
-    runs-on: macos-12
+    runs-on: self-hosted
-
+    strategy:
+      matrix:
+        scenario:
+          - default
+          - ipv6
+          - single_node
+          - calico
+          - kube-vip
+      fail-fast: false
     env:
-      HOMEBREW_NO_INSTALL_CLEANUP: 1
+      PYTHON_VERSION: "3.11"
-      VAGRANT_CWD: ${{ github.workspace }}/vagrant
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Install Ansible
+      # these steps are necessary if not using ephemeral nodes
-        run: brew install ansible
+      - name: Delete old Vagrant box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force
 
-      - name: Install role dependencies
+      - name: Remove all local Vagrant boxes
-        run: ansible-galaxy install -r collections/requirements.yml
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
 
       - name: Configure VirtualBox
-        run: >-
+        run: |-
-          sudo mkdir -p /etc/vbox &&
+          sudo mkdir -p /etc/vbox
-          echo "* 192.168.30.0/24" | sudo tee -a /etc/vbox/networks.conf > /dev/null
+          cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
+          * 192.168.30.0/24
+          * fdad:bad:ba55::/64
+          EOF
 
-      - name: Cache Vagrant boxes
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/cache@fd5de65bc895cf536527842281bea11763fefd77 # 3.0.8
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
-          path: |
+          python-version: ${{ env.PYTHON_VERSION }}
-            ~/.vagrant.d/boxes
+          cache: 'pip' # caching pip dependencies
-          key: vagrant-boxes-${{ hashFiles('**/Vagrantfile') }}
-          restore-keys: |
-            vagrant-boxes
 
-      - name: Create virtual machines
+      - name: Restore vagrant Boxes cache
-        run: vagrant up
+        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
-        timeout-minutes: 10
+        with:
+          path: ~/.vagrant.d/boxes
+          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+          fail-on-cache-miss: true
 
-      - name: Provision cluster using Ansible
+      - name: Install dependencies
-        # Since Ansible sets up _all_ machines, it is sufficient to run it only
+        run: |
-        # once (i.e, for a single node - we are choosing control1 here)
+          echo "::group::Upgrade pip"
-        run: vagrant provision control1 --provision-with ansible
+          python3 -m pip install --upgrade pip
-        timeout-minutes: 25
+          echo "::endgroup::"
 
-      - name: Set up kubectl on the host
+          echo "::group::Install Python requirements from requirements.txt"
-        run: brew install kubectl &&
+          python3 -m pip install -r requirements.txt
-          mkdir -p ~/.kube &&
+          echo "::endgroup::"
-          vagrant ssh control1 --command "cat ~/.kube/config" > ~/.kube/config
 
-      - name: Show cluster nodes
+      - name: Test with molecule
-        run: kubectl describe -A nodes
+        run: molecule test --scenario-name ${{ matrix.scenario }}
+        timeout-minutes: 90
+        env:
+          ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
+          ANSIBLE_SSH_RETRIES: 4
+          ANSIBLE_TIMEOUT: 120
+          PY_COLORS: 1
+          ANSIBLE_FORCE_COLOR: 1
 
-      - name: Show cluster pods
+      # these steps are necessary if not using ephemeral nodes
-        run: kubectl describe -A pods
+      - name: Delete old Vagrant box versions
-
-      - name: Test cluster
-        run: $VAGRANT_CWD/test_cluster.py --verbose --locals
-        timeout-minutes: 5
-
-      - name: Destroy virtual machines
         if: always() # do this even if a step before has failed
-        run: vagrant destroy --force
+        run: vagrant box prune --force
+
+      - name: Remove all local Vagrant boxes
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
+
+      - name: Upload log files
+        if: always() # do this even if a step before has failed
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # 4.3.0
+        with:
+          name: logs
+          path: |
+            ${{ runner.temp }}/logs
+          overwrite: true

.gitignore

@@ -1 +1,4 @@
-.vagrant
+.env/
+*.log
+ansible.cfg
+kubeconfig

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: f71fa2c1f9cf5cb705f73dffe4b21f7c61470ba9  # frozen: v4.4.0
+    hooks:
+      - id: requirements-txt-fixer
+      - id: sort-simple-yaml
+      - id: detect-private-key
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: b05e028c5881819161d11cb543fd96a30c06cceb  # frozen: v1.32.0
+    hooks:
+      - id: yamllint
+        args: [-c=.yamllint]
+  - repo: https://github.com/ansible-community/ansible-lint.git
+    rev: 3293b64b939c0de16ef8cb81dd49255e475bf89a  # frozen: v6.17.2
+    hooks:
+      - id: ansible-lint
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: 375289a39f5708101b1f916eb729e8d6da96993f  # frozen: v0.9.0.5
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: 12885e376b93dc4536ad68d156065601e4433665  # frozen: v1.5.1
+    hooks:
+      - id: remove-crlf
+      - id: remove-tabs
+  - repo: https://github.com/sirosen/texthooks
+    rev: c4ffd3e31669dd4fa4d31a23436cc13839730084  # frozen: 0.5.0
+    hooks:
+      - id: fix-smartquotes

.yamllint

@@ -6,4 +6,6 @@ rules:
     max: 120
     level: warning
   truthy:
-    allowed-values: ['true', 'false', 'yes', 'no']
+    allowed-values: ['true', 'false']
+ignore:
+  - galaxy.yml

README.md

@@ -4,21 +4,21 @@
 
 This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`.
 
-This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.chipzoller.dev/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
+This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
 
 If you want more context on how this works, see:
 
-📄 [Documentation](https://docs.technotim.live/posts/k3s-etcd-ansible/) (including example commands)
+📄 [Documentation](https://technotim.live/posts/k3s-etcd-ansible/) (including example commands)
 
-📺 [Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
+📺 [Watch the Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
 
 ## 📖 k3s Ansible Playbook
 
 Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running:
 
-- [X] Debian
+- [x] Debian (tested on version 11)
-- [X] Ubuntu
+- [x] Ubuntu (tested on version 22.04)
-- [X] CentOS
+- [x] Rocky (tested on version 9)
 
 on processor architecture:
 
@@ -28,7 +28,12 @@ on processor architecture:
 
 ## ✅ System requirements
 
-- Deployment environment must have Ansible 2.4.0+.  If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://docs.technotim.live/posts/ansible-automation/).
+- Control Node (the machine you are running `ansible` commands) must have Ansible 2.11+ If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://technotim.live/posts/ansible-automation/).
+
+- You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml` (important❗)
+
+- [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
+
 - `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command.
 
 ## 🚀 Getting Started
@@ -62,6 +67,8 @@ node
 
 If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/).
 
+Finally, copy `ansible.example.cfg` to `ansible.cfg` and adapt the inventory path to match the files that you just created.
+
 This requires at least k3s version `1.19.1` however the version is configurable by using the `k3s_version` variable.
 
 If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
@@ -94,24 +101,48 @@ scp debian@master_ip:~/.kube/config ~/.kube/config
 
 ### 🔨 Testing your cluster
 
-See the commands [here](https://docs.technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
+See the commands [here](https://technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
 
 ### Troubleshooting
 
 Be sure to see [this post](https://github.com/techno-tim/k3s-ansible/discussions/20) on how to troubleshoot common problems
 
-### 🔷 Vagrant
+### Testing the playbook using molecule
 
-You may want to kickstart your k3s cluster by using Vagrant to quickly build you all needed VMs with one command.
+This playbook includes a [molecule](https://molecule.rtfd.io/)-based test setup.
-Head to the `vagrant` subfolder and type `vagrant up` to get your environment setup.
+It is run automatically in CI, but you can also run the tests locally.
-After the VMs have got build, deploy k3s using the Ansible playbook `site.yml` by the
+This might be helpful for quick feedback in a few cases.
-`vagrant provision --provision-with ansible` command.
+You can find more information about it [here](molecule/README.md).
+
+### Pre-commit Hooks
+
+This repo uses `pre-commit` and `pre-commit-hooks` to lint and fix common style and syntax errors.  Be sure to install python packages and then run `pre-commit install`.  For more information, see [pre-commit](https://pre-commit.com/)
+
+## 🌌 Ansible Galaxy
+
+This collection can now be used in larger ansible projects.
+
+Instructions:
+
+- create or modify a file `collections/requirements.yml` in your project
+
+```yml
+collections:
+  - name: ansible.utils
+  - name: community.general
+  - name: ansible.posix
+  - name: kubernetes.core
+  - name: https://github.com/techno-tim/k3s-ansible.git
+    type: git
+    version: master
+```
+
+- install via `ansible-galaxy collection install -r ./collections/requirements.yml`
+- every role is now available via the prefix `techno_tim.k3s_ansible.` e.g. `techno_tim.k3s_ansible.lxc`
 
 ## Thanks 🤝
 
-This repo is really standing on the shoulders of giants.  To all those who have contributed.
+This repo is really standing on the shoulders of giants. Thank you to all those who have contributed and thanks to these repos for code and ideas:
-
-Thanks to these repos for code and ideas:
 
 - [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible)
 - [geerlingguy/turing-pi-cluster](https://github.com/geerlingguy/turing-pi-cluster)

ansible.cfg

@@ -1,12 +0,0 @@
-[defaults]
-nocows = True
-roles_path = ./roles
-inventory  = ./hosts.ini
-
-remote_tmp = $HOME/.ansible/tmp
-local_tmp  = $HOME/.ansible/tmp
-pipelining = True
-become = True
-host_key_checking = False
-deprecation_warnings = False
-callback_whitelist = profile_tasks

ansible.example.cfg

@@ -0,0 +1,2 @@
+[defaults]
+inventory = inventory/my-cluster/hosts.ini  ; Adapt this to the path to your inventory file

collections/requirements.yml

@@ -1,4 +1,6 @@
 ---
 collections:
+  - name: ansible.utils
   - name: community.general
   - name: ansible.posix
+  - name: kubernetes.core

deploy.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-ansible-playbook site.yml -i inventory/my-cluster/hosts.ini
+ansible-playbook site.yml

example/service.yml

@@ -4,6 +4,7 @@ kind: Service
 metadata:
   name: nginx
 spec:
+  ipFamilyPolicy: PreferDualStack
   selector:
     app: nginx
   ports:

galaxy.yml

@@ -0,0 +1,81 @@
+### REQUIRED
+# The namespace of the collection. This can be a company/brand/organization or product namespace under which all
+# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with
+# underscores or numbers and cannot contain consecutive underscores
+namespace: techno_tim
+
+# The name of the collection. Has the same character restrictions as 'namespace'
+name: k3s_ansible
+
+# The version of the collection. Must be compatible with semantic versioning
+version: 1.0.0
+
+# The path to the Markdown (.md) readme file. This path is relative to the root of the collection
+readme: README.md
+
+# A list of the collection's content authors. Can be just the name or in the format 'Full Name <email> (url)
+# @nicks:irc/im.site#channel'
+authors:
+- your name <example@domain.com>
+
+
+### OPTIONAL but strongly recommended
+# A short summary description of the collection
+description: >
+  The easiest way to bootstrap a self-hosted High Availability Kubernetes
+  cluster. A fully automated HA k3s etcd install with kube-vip, MetalLB,
+  and more.
+
+# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only
+# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file'
+license:
+- Apache-2.0
+
+
+# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character
+# requirements as 'namespace' and 'name'
+tags:
+  - etcd
+  - high-availability
+  - k8s
+  - k3s
+  - k3s-cluster
+  - kube-vip
+  - kubernetes
+  - metallb
+  - rancher
+
+# Collections that this collection requires to be installed for it to be usable. The key of the dict is the
+# collection label 'namespace.name'. The value is a version range
+# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
+# range specifiers can be set and are separated by ','
+dependencies:
+  ansible.utils: '*'
+  ansible.posix: '*'
+  community.general: '*'
+  kubernetes.core: '*'
+
+# The URL of the originating SCM repository
+repository: https://github.com/techno-tim/k3s-ansible
+
+# The URL to any online docs
+documentation: https://github.com/techno-tim/k3s-ansible
+
+# The URL to the homepage of the collection/project
+homepage: https://www.youtube.com/watch?v=CbkEWcUZ7zM
+
+# The URL to the collection issue tracker
+issues: https://github.com/techno-tim/k3s-ansible/issues
+
+# A list of file glob-like patterns used to filter any files or directories that should not be included in the build
+# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
+# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry',
+# and '.git' are always filtered. Mutually exclusive with 'manifest'
+build_ignore: []
+
+# A dict controlling use of manifest directives used in building the collection artifact. The key 'directives' is a
+# list of MANIFEST.in style
+# L(directives,https://packaging.python.org/en/latest/guides/using-manifest-in/#manifest-in-commands). The key
+# 'omit_default_directives' is a boolean that controls whether the default directives are used. Mutually exclusive
+# with 'build_ignore'
+# manifest: null

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.24.4+k3s1
+k3s_version: v1.29.0+k3s1
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -10,6 +10,12 @@ system_timezone: "Your/Timezone"
 # interface which will be used for flannel
 flannel_iface: "eth0"
 
+# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
+# calico_iface: "eth0"
+calico_ebpf: false           # use eBPF dataplane instead of iptables
+calico_cidr: "10.52.0.0/16"  # calico cluster pod cidr pool
+calico_tag: "v3.27.0"        # calico version tag
+
 # apiserver_endpoint is virtual ip-address which will be configured on each master
 apiserver_endpoint: "192.168.30.222"
 
@@ -17,16 +23,127 @@ apiserver_endpoint: "192.168.30.222"
 # this token should be alpha numeric only
 k3s_token: "some-SUPER-DEDEUPER-secret-password"
 
-# change these to your liking, the only required one is--disable servicelb
+# The IP on which the node is reachable in the cluster.
-extra_server_args: "--disable servicelb --disable traefik"
+# Here, a sensible default is provided, you can still override
-extra_agent_args: ""
+# it for each of your hosts, though.
+k3s_node_ip: "{{ ansible_facts[(calico_iface | default(flannel_iface))]['ipv4']['address'] }}"
+
+# Disable the taint manually by setting: k3s_master_taint = false
+k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
+
+# these arguments are recommended for servers as well as agents:
+extra_args: >-
+  {{ '--flannel-iface=' + flannel_iface if calico_iface is not defined else '' }}
+  --node-ip={{ k3s_node_ip }}
+
+# change these to your liking, the only required are: --disable servicelb, --tls-san {{ apiserver_endpoint }}
+# the contents of the if block is also required if using calico
+extra_server_args: >-
+  {{ extra_args }}
+  {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
+  {% if calico_iface is defined %}
+  --flannel-backend=none
+  --disable-network-policy
+  --cluster-cidr={{ calico_cidr | default('10.52.0.0/16') }}
+  {% endif %}
+  --tls-san {{ apiserver_endpoint }}
+  --disable servicelb
+  --disable traefik
+
+extra_agent_args: >-
+  {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.0"
+kube_vip_tag_version: "v0.6.4"
+
+# tag for kube-vip-cloud-provider manifest
+# kube_vip_cloud_provider_tag_version: "main"
+
+# kube-vip ip range for load balancer
+# (uncomment to use kube-vip for services instead of MetalLB)
+# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
+
+# metallb type frr or native
+metal_lb_type: "native"
+
+# metallb mode layer2 or bgp
+metal_lb_mode: "layer2"
+
+# bgp options
+# metal_lb_bgp_my_asn: "64513"
+# metal_lb_bgp_peer_asn: "64512"
+# metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.5"
+metal_lb_speaker_tag_version: "v0.13.12"
-metal_lb_controller_tag_version: "v0.13.5"
+metal_lb_controller_tag_version: "v0.13.12"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
+
+# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
+# in your hosts.ini file.
+# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
+# Most notably, your containers must be privileged, and must not have nesting set to true.
+# Please note this script disables most of the security of lxc containers, with the trade off being that lxc
+# containers are significantly more resource efficient compared to full VMs.
+# Mixing and matching VMs and lxc containers is not supported, ymmv if you want to do this.
+# I would only really recommend using this if you have particularly low powered proxmox nodes where the overhead of
+# VMs would use a significant portion of your available resources.
+proxmox_lxc_configure: false
+# the user that you would use to ssh into the host, for example if you run ssh some-user@my-proxmox-host,
+# set this value to some-user
+proxmox_lxc_ssh_user: root
+# the unique proxmox ids for all of the containers in the cluster, both worker and master nodes
+proxmox_lxc_ct_ids:
+  - 200
+  - 201
+  - 202
+  - 203
+  - 204
+
+# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache
+# (harbor / nexus / docker's official registry / etc).
+# Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub),
+# or air-gapped environments where your nodes don't have internet access after the initial setup
+# (which is still needed for downloading the k3s binary and such).
+# k3s's documentation about private registries here: https://docs.k3s.io/installation/private-registry
+custom_registries: false
+# The registries can be authenticated or anonymous, depending on your registry server configuration.
+# If they allow anonymous access, simply remove the following bit from custom_registries_yaml
+#   configs:
+#     "registry.domain.com":
+#       auth:
+#         username: yourusername
+#         password: yourpassword
+# The following is an example that pulls all images used in this playbook through your private registries.
+# It also allows you to pull your own images from your private registry, without having to use imagePullSecrets
+# in your deployments.
+# If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images,
+# you can just remove those from the mirrors: section.
+custom_registries_yaml: |
+  mirrors:
+    docker.io:
+      endpoint:
+        - "https://registry.domain.com/v2/dockerhub"
+    quay.io:
+      endpoint:
+        - "https://registry.domain.com/v2/quayio"
+    ghcr.io:
+      endpoint:
+        - "https://registry.domain.com/v2/ghcrio"
+    registry.domain.com:
+      endpoint:
+        - "https://registry.domain.com"
+
+  configs:
+    "registry.domain.com":
+      auth:
+        username: yourusername
+        password: yourpassword
+
+# Only enable and configure these if you access the internet through a proxy
+# proxy_env:
+#   HTTP_PROXY: "http://proxy.domain.local:3128"
+#   HTTPS_PROXY: "http://proxy.domain.local:3128"
+#   NO_PROXY: "*.domain.local,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"

inventory/sample/group_vars/proxmox.yml

@@ -0,0 +1,2 @@
+---
+ansible_user: '{{ proxmox_lxc_ssh_user }}'

inventory/sample/hosts.ini

@@ -7,6 +7,11 @@
 192.168.30.41
 192.168.30.42
 
+# only required if proxmox_lxc_configure: true
+# must contain all proxmox instances that have a master or worker node
+# [proxmox]
+# 192.168.30.43
+
 [k3s_cluster:children]
 master
 node

molecule/README.md

@@ -0,0 +1,77 @@
+# Test suites for `k3s-ansible`
+
+This folder contains the [molecule](https://molecule.rtfd.io/)-based test setup for this playbook.
+
+## Scenarios
+
+We have these scenarios:
+
+- **default**:
+  A 3 control + 2 worker node cluster based very closely on the [sample inventory](../inventory/sample/).
+- **ipv6**:
+  A cluster that is externally accessible via IPv6 ([more information](ipv6/README.md))
+  To save a bit of test time, this cluster is _not_ highly available, it consists of only one control and one worker node.
+- **single_node**:
+  Very similar to the default scenario, but uses only a single node for all cluster functionality.
+- **calico**:
+  The same as single node, but uses calico cni instead of flannel.
+- **kube-vip**
+  The same as single node, but uses kube-vip as service loadbalancer instead of MetalLB
+
+## How to execute
+
+To test on your local machine, follow these steps:
+
+### System requirements
+
+Make sure that the following software packages are available on your system:
+
+- [Python 3](https://www.python.org/downloads)
+- [Vagrant](https://www.vagrantup.com/downloads)
+- [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+
+### Set up VirtualBox networking on Linux and macOS
+
+_You can safely skip this if you are working on Windows._
+
+Furthermore, the test cluster uses the `192.168.30.0/24` subnet which is [not set up by VirtualBox automatically](https://www.virtualbox.org/manual/ch06.html#network_hostonly).
+To set the subnet up for use with VirtualBox, please make sure that `/etc/vbox/networks.conf` exists and that it contains this line:
+
+```
+* 192.168.30.0/24
+* fdad:bad:ba55::/64
+```
+
+### Install Python dependencies
+
+You will get [Molecule, Ansible and a few extra dependencies](../requirements.txt) via [pip](https://pip.pypa.io/).
+Usually, it is advisable to work in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for this:
+
+```bash
+cd /path/to/k3s-ansible
+
+# Create a virtualenv at ".env". You only need to do this once.
+python3 -m venv .env
+
+# Activate the virtualenv for your current shell session.
+# If you start a new session, you will have to repeat this.
+source .env/bin/activate
+
+# Install the required packages into the virtualenv.
+# These remain installed across shell sessions.
+python3 -m pip install -r requirements.txt
+```
+
+### Run molecule
+
+With the virtual environment from the previous step active in your shell session, you can now use molecule to test the playbook.
+Interesting commands are:
+
+- `molecule create`: Create virtual machines for the test cluster nodes.
+- `molecule destroy`: Delete the virtual machines for the test cluster nodes.
+- `molecule converge`: Run the `site` playbook on the nodes of the test cluster.
+- `molecule side_effect`: Run the `reset` playbook on the nodes of the test cluster.
+- `molecule verify`: Verify that the cluster works correctly.
+- `molecule test`: The "all-in-one" sequence of steps that is executed in CI.
+  This includes the `create`, `converge`, `verify`, `side_effect` and `destroy` steps.
+  See [`molecule.yml`](default/molecule.yml) for more details.

molecule/calico/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/calico/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        calico_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.224"
+        metal_lb_ip_range: "192.168.30.100-192.168.30.109"

molecule/default/molecule.yml

@@ -0,0 +1,100 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.38
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: control2
+    box: generic/debian12
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.39
+
+  - name: control3
+    box: generic/rocky9
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.40
+
+  - name: node1
+    box: generic/ubuntu2204
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.41
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: node2
+    box: generic/rocky9
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.42
+
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/default/overrides.yml

@@ -0,0 +1,12 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45

molecule/default/prepare.yml

@@ -0,0 +1,22 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
+
+- name: Network setup
+  hosts: all
+  tasks:
+    - name: Disable firewalld
+      when: ansible_distribution == "Rocky"
+      # Rocky Linux comes with firewalld enabled. It blocks some of the network
+      # connections needed for our k3s cluster. For our test setup, we just disable
+      # it since the VM host's firewall is still active for connections to and from
+      # the Internet.
+      # When building your own cluster, please DO NOT blindly copy this. Instead,
+      # please create a custom firewall configuration that fits your network design
+      # and security needs.
+      ansible.builtin.systemd:
+        name: firewalld
+        enabled: false
+        state: stopped
+      become: true

molecule/ipv6/README.md

@@ -0,0 +1,35 @@
+# Sample IPv6 configuration for `k3s-ansible`
+
+This scenario contains a cluster configuration which is _IPv6 first_, but still supports dual-stack networking with IPv4 for most things.
+This means:
+
+- The API server VIP is an IPv6 address.
+- The MetalLB pool consists of both IPv4 and IPv4 addresses.
+- Nodes as well as cluster-internal resources (pods and services) are accessible via IPv4 as well as IPv6.
+
+## Network design
+
+All IPv6 addresses used in this scenario share a single `/48` prefix: `fdad:bad:ba55`.
+The following subnets are used:
+
+- `fdad:bad:ba55:`**`0`**`::/64` is the subnet which contains the cluster components meant for external access.
+  That includes:
+
+  - The VIP for the Kubernetes API server: `fdad:bad:ba55::333`
+  - Services load-balanced by MetalLB: `fdad:bad:ba55::1b:0/112`
+  - Cluster nodes: `fdad:bad:ba55::de:0/112`
+  - The host executing Vagrant: `fdad:bad:ba55::1`
+
+  In a home lab setup, this might be your LAN.
+
+- `fdad:bad:ba55:`**`4200`**`::/56` is used internally by the cluster for pods.
+
+- `fdad:bad:ba55:`**`4300`**`::/108` is used internally by the cluster for services.
+
+IPv4 networking is also available:
+
+- The nodes have addresses inside `192.168.123.0/24`.
+  MetalLB also has a bit of address space in this range: `192.168.123.80-192.168.123.90`
+- For pods and services, the k3s defaults (`10.42.0.0/16` and `10.43.0.0/16)` are used.
+
+Note that the host running Vagrant is not part any of these IPv4 networks.

molecule/ipv6/host_vars/control1.yml

@@ -0,0 +1,3 @@
+---
+node_ipv4: 192.168.123.11
+node_ipv6: fdad:bad:ba55::de:11

molecule/ipv6/host_vars/control2.yml

@@ -0,0 +1,3 @@
+---
+node_ipv4: 192.168.123.12
+node_ipv6: fdad:bad:ba55::de:12

molecule/ipv6/host_vars/node1.yml

@@ -0,0 +1,3 @@
+---
+node_ipv4: 192.168.123.21
+node_ipv6: fdad:bad:ba55::de:21

molecule/ipv6/molecule.yml

@@ -0,0 +1,81 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: fdad:bad:ba55::de:11
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: control2
+    box: generic/ubuntu2204
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: fdad:bad:ba55::de:12
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: node1
+    box: generic/ubuntu2204
+    memory: 1024
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
+    interfaces:
+      - network_name: private_network
+        ip: fdad:bad:ba55::de:21
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/ipv6/overrides.yml

@@ -0,0 +1,51 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables (1/2)
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # In this scenario, we have multiple interfaces that the VIP could be
+        # broadcasted on. Since we have assigned a dedicated private network
+        # here, let's make sure that it is used.
+        kube_vip_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # IPv6 configuration
+        # ######################################################################
+
+        # The API server will be reachable on IPv6 only
+        apiserver_endpoint: fdad:bad:ba55::333
+
+        # We give MetalLB address space for both IPv4 and IPv6
+        metal_lb_ip_range:
+          - fdad:bad:ba55::1b:0/112
+          - 192.168.123.80-192.168.123.90
+
+        # k3s_node_ip is by default set to the IPv4 address of flannel_iface.
+        # We want IPv6 addresses here of course, so we just specify them
+        # manually below.
+        k3s_node_ip: "{{ node_ipv4 }},{{ node_ipv6 }}"
+
+    - name: Override host variables (2/2)
+      # Since "extra_args" depends on "k3s_node_ip" and "flannel_iface" we have
+      # to set this AFTER overriding the both of them.
+      ansible.builtin.set_fact:
+        # A few extra server args are necessary:
+        #  - the network policy needs to be disabled.
+        #  - we need to manually specify the subnets for services and pods, as
+        #    the default has IPv4 ranges only.
+        extra_server_args: >-
+          {{ extra_args }}
+          --tls-san {{ apiserver_endpoint }}
+          {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
+          --disable servicelb
+          --disable traefik
+          --disable-network-policy
+          --cluster-cidr=10.42.0.0/16,fdad:bad:ba55:4200::/56
+          --service-cidr=10.43.0.0/16,fdad:bad:ba55:4300::/108

molecule/ipv6/prepare.yml

@@ -0,0 +1,51 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
+
+- name: Configure dual-stack networking
+  hosts: all
+  become: true
+
+  # Unfortunately, as of 2022-09, Vagrant does not support the configuration
+  # of both IPv4 and IPv6 addresses for a single network adapter. So we have
+  # to configure that ourselves.
+  # Moreover, we have to explicitly enable IPv6 for the loopback interface.
+
+  tasks:
+    - name: Enable IPv6 for network interfaces
+      ansible.posix.sysctl:
+        name: net.ipv6.conf.{{ item }}.disable_ipv6
+        value: "0"
+      with_items:
+        - all
+        - default
+        - lo
+
+    - name: Disable duplicate address detection
+      # Duplicate address detection did repeatedly fail within the virtual
+      # network. But since this setup does not use SLAAC anyway, we can safely
+      # disable it.
+      ansible.posix.sysctl:
+        name: net.ipv6.conf.{{ item }}.accept_dad
+        value: "0"
+      with_items:
+        - "{{ flannel_iface }}"
+
+    - name: Write IPv4 configuration
+      ansible.builtin.template:
+        src: 55-flannel-ipv4.yaml.j2
+        dest: /etc/netplan/55-flannel-ipv4.yaml
+        owner: root
+        group: root
+        mode: 0644
+      register: netplan_template
+
+    - name: Apply netplan configuration
+      # Conceptually, this should be a handler rather than a task.
+      # However, we are currently not in a role context - creating
+      # one just for this seemed overkill.
+      when: netplan_template.changed
+      ansible.builtin.command:
+        cmd: netplan apply
+      changed_when: true

molecule/ipv6/templates/55-flannel-ipv4.yaml.j2

@@ -0,0 +1,8 @@
+---
+network:
+  version: 2
+  renderer: networkd
+  ethernets:
+    {{ flannel_iface }}:
+      addresses:
+      - {{ node_ipv4 }}/24

molecule/kube-vip/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/kube-vip/overrides.yml

@@ -0,0 +1,17 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.225"
+        # Use kube-vip instead of MetalLB
+        kube_vip_lb_ip_range: "192.168.30.110-192.168.30.119"

molecule/resources/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
+
+- name: Converge
+  ansible.builtin.import_playbook: ../../site.yml

molecule/resources/reset.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
+
+- name: Reset
+  ansible.builtin.import_playbook: ../../reset.yml

molecule/resources/verify.yml

@@ -0,0 +1,5 @@
+---
+- name: Verify
+  hosts: all
+  roles:
+    - verify_from_outside

molecule/resources/verify_from_outside/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# A host outside of the cluster from which the checks shall be performed
+outside_host: localhost
+
+# This kubernetes namespace will be used for testing
+testing_namespace: molecule-verify-from-outside
+
+# The directory in which the example manifests reside
+example_manifests_path: ../../../example

molecule/resources/verify_from_outside/tasks/kubecfg-cleanup.yml

@@ -0,0 +1,5 @@
+---
+- name: Clean up kubecfg
+  ansible.builtin.file:
+    path: "{{ kubecfg.path }}"
+    state: absent

molecule/resources/verify_from_outside/tasks/kubecfg-fetch.yml

@@ -0,0 +1,19 @@
+---
+- name: Create temporary directory for kubecfg
+  ansible.builtin.tempfile:
+    state: directory
+    suffix: kubecfg
+  register: kubecfg
+- name: Gathering facts
+  delegate_to: "{{ groups['master'][0] }}"
+  ansible.builtin.gather_facts:
+- name: Download kubecfg
+  ansible.builtin.fetch:
+    src: "{{ ansible_env.HOME }}/.kube/config"
+    dest: "{{ kubecfg.path }}/"
+    flat: true
+  delegate_to: "{{ groups['master'][0] }}"
+  delegate_facts: true
+- name: Store path to kubecfg
+  ansible.builtin.set_fact:
+    kubecfg_path: "{{ kubecfg.path }}/config"

molecule/resources/verify_from_outside/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- name: Verify
+  run_once: true
+  delegate_to: "{{ outside_host }}"
+  block:
+    - name: "Test CASE: Get kube config"
+      ansible.builtin.import_tasks: kubecfg-fetch.yml
+    - name: "TEST CASE: Get nodes"
+      ansible.builtin.include_tasks: test/get-nodes.yml
+    - name: "TEST CASE: Deploy example"
+      ansible.builtin.include_tasks: test/deploy-example.yml
+  always:
+    - name: "TEST CASE: Cleanup"
+      ansible.builtin.import_tasks: kubecfg-cleanup.yml

molecule/resources/verify_from_outside/tasks/test/deploy-example.yml

@@ -0,0 +1,58 @@
+---
+- name: Deploy example
+  block:
+    - name: "Create namespace: {{ testing_namespace }}"
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: "{{ testing_namespace }}"
+        state: present
+        wait: true
+        kubeconfig: "{{ kubecfg_path }}"
+
+    - name: Apply example manifests
+      kubernetes.core.k8s:
+        src: "{{ example_manifests_path }}/{{ item }}"
+        namespace: "{{ testing_namespace }}"
+        state: present
+        wait: true
+        kubeconfig: "{{ kubecfg_path }}"
+      with_items:
+        - deployment.yml
+        - service.yml
+
+    - name: Get info about nginx service
+      kubernetes.core.k8s_info:
+        kind: service
+        name: nginx
+        namespace: "{{ testing_namespace }}"
+        kubeconfig: "{{ kubecfg_path }}"
+      vars: &load_balancer_metadata
+        metallb_ip: status.loadBalancer.ingress[0].ip
+        metallb_port: spec.ports[0].port
+      register: nginx_services
+
+    - name: Assert that the nginx welcome page is available
+      ansible.builtin.uri:
+        url: http://{{ ip | ansible.utils.ipwrap }}:{{ port_ }}/
+        return_content: true
+      register: result
+      failed_when: "'Welcome to nginx!' not in result.content"
+      vars:
+        ip: >-
+          {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
+        port_: >-
+          {{ nginx_services.resources[0].spec.ports[0].port }}
+      # Deactivated linter rules:
+      #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+      #                     would be undefined. This will not be the case during playbook execution.
+      # noqa jinja[invalid]
+
+  always:
+    - name: "Remove namespace: {{ testing_namespace }}"
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: "{{ testing_namespace }}"
+        state: absent
+        kubeconfig: "{{ kubecfg_path }}"

molecule/resources/verify_from_outside/tasks/test/get-nodes.yml

@@ -0,0 +1,28 @@
+---
+- name: Get all nodes in cluster
+  kubernetes.core.k8s_info:
+    kind: node
+    kubeconfig: "{{ kubecfg_path }}"
+  register: cluster_nodes
+
+- name: Assert that the cluster contains exactly the expected nodes
+  ansible.builtin.assert:
+    that: found_nodes == expected_nodes
+    success_msg: "Found nodes as expected: {{ found_nodes }}"
+    fail_msg: "Expected nodes {{ expected_nodes }}, but found nodes {{ found_nodes }}"
+  vars:
+    found_nodes: >-
+      {{ cluster_nodes | json_query('resources[*].metadata.name') | unique | sort }}
+    expected_nodes: |-
+      {{
+        (
+          ( groups['master'] | default([]) ) +
+          ( groups['node'] | default([]) )
+        )
+        | unique
+        | sort
+      }}
+  # Deactivated linter rules:
+  #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+  #                     would be undefined. This will not be the case during playbook execution.
+  # noqa jinja[invalid]

molecule/single_node/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.50
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/single_node/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the default scenario
+        apiserver_endpoint: "192.168.30.223"
+        metal_lb_ip_range: "192.168.30.91-192.168.30.99"

reboot.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ansible-playbook reboot.yml

reboot.yml

@@ -0,0 +1,9 @@
+---
+- name: Reboot k3s_cluster
+  hosts: k3s_cluster
+  gather_facts: true
+  tasks:
+    - name: Reboot the nodes (and Wait upto 5 mins max)
+      become: true
+      reboot:
+        reboot_timeout: 300

requirements.in

@@ -0,0 +1,10 @@
+ansible-core>=2.16.2
+jmespath>=1.0.1
+jsonpatch>=1.33
+kubernetes>=29.0.0
+molecule-plugins[vagrant]
+molecule>=6.0.3
+netaddr>=0.10.1
+pre-commit>=3.6.0
+pre-commit-hooks>=4.5.0
+pyyaml>=6.0.1

requirements.txt

@@ -0,0 +1,169 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+ansible-compat==4.1.11
+    # via molecule
+ansible-core==2.16.2
+    # via
+    #   -r requirements.in
+    #   ansible-compat
+    #   molecule
+attrs==23.2.0
+    # via
+    #   jsonschema
+    #   referencing
+bracex==2.4
+    # via wcmatch
+cachetools==5.3.2
+    # via google-auth
+certifi==2023.11.17
+    # via
+    #   kubernetes
+    #   requests
+cffi==1.16.0
+    # via cryptography
+cfgv==3.4.0
+    # via pre-commit
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   click-help-colors
+    #   molecule
+click-help-colors==0.9.4
+    # via molecule
+cryptography==41.0.7
+    # via ansible-core
+distlib==0.3.8
+    # via virtualenv
+enrich==1.2.7
+    # via molecule
+filelock==3.13.1
+    # via virtualenv
+google-auth==2.26.2
+    # via kubernetes
+identify==2.5.33
+    # via pre-commit
+idna==3.6
+    # via requests
+jinja2==3.1.3
+    # via
+    #   ansible-core
+    #   molecule
+jmespath==1.0.1
+    # via -r requirements.in
+jsonpatch==1.33
+    # via -r requirements.in
+jsonpointer==2.4
+    # via jsonpatch
+jsonschema==4.21.1
+    # via
+    #   ansible-compat
+    #   molecule
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+kubernetes==29.0.0
+    # via -r requirements.in
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.4
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+molecule==6.0.3
+    # via
+    #   -r requirements.in
+    #   molecule-plugins
+molecule-plugins[vagrant]==23.5.0
+    # via -r requirements.in
+netaddr==0.10.1
+    # via -r requirements.in
+nodeenv==1.8.0
+    # via pre-commit
+oauthlib==3.2.2
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+packaging==23.2
+    # via
+    #   ansible-compat
+    #   ansible-core
+    #   molecule
+platformdirs==4.1.0
+    # via virtualenv
+pluggy==1.3.0
+    # via molecule
+pre-commit==3.6.0
+    # via -r requirements.in
+pre-commit-hooks==4.5.0
+    # via -r requirements.in
+pyasn1==0.5.1
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+pycparser==2.21
+    # via cffi
+pygments==2.17.2
+    # via rich
+python-dateutil==2.8.2
+    # via kubernetes
+python-vagrant==1.0.0
+    # via molecule-plugins
+pyyaml==6.0.1
+    # via
+    #   -r requirements.in
+    #   ansible-compat
+    #   ansible-core
+    #   kubernetes
+    #   molecule
+    #   pre-commit
+referencing==0.32.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.31.0
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+requests-oauthlib==1.3.1
+    # via kubernetes
+resolvelib==1.0.1
+    # via ansible-core
+rich==13.7.0
+    # via
+    #   enrich
+    #   molecule
+rpds-py==0.17.1
+    # via
+    #   jsonschema
+    #   referencing
+rsa==4.9
+    # via google-auth
+ruamel-yaml==0.18.5
+    # via pre-commit-hooks
+ruamel-yaml-clib==0.2.8
+    # via ruamel-yaml
+six==1.16.0
+    # via
+    #   kubernetes
+    #   python-dateutil
+subprocess-tee==0.4.1
+    # via ansible-compat
+urllib3==2.1.0
+    # via
+    #   kubernetes
+    #   requests
+virtualenv==20.25.0
+    # via pre-commit
+wcmatch==8.5
+    # via molecule
+websocket-client==1.7.0
+    # via kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

reset.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
+ansible-playbook reset.yml

reset.yml

@@ -1,7 +1,24 @@
 ---
-
+- name: Reset k3s cluster
-- hosts: k3s_cluster
+  hosts: k3s_cluster
-  gather_facts: yes
+  gather_facts: true
-  become: yes
   roles:
     - role: reset
+      become: true
+    - role: raspberrypi
+      become: true
+      vars: {state: absent}
+  post_tasks:
+    - name: Reboot and wait for node to come back up
+      become: true
+      reboot:
+        reboot_timeout: 3600
+
+- name: Revert changes to Proxmox cluster
+  hosts: proxmox
+  gather_facts: true
+  become: true
+  remote_user: "{{ proxmox_lxc_ssh_user }}"
+  roles:
+    - role: reset_proxmox_lxc
+      when: proxmox_lxc_configure

roles/k3s/master/defaults/main.yml

@@ -1,12 +0,0 @@
----
-ansible_user: root
-server_init_args: >-
-  {% if groups['master'] | length > 1 %}
-    {% if ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) %}
-      --cluster-init
-    {% else %}
-      --server https://{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}:6443
-    {% endif %}
-    --token {{ k3s_token }}
-  {% endif %}
-  {{ extra_server_args | default('') }}

roles/k3s/master/templates/metallb.namespace.j2

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: metallb-system
-  labels:
-    app: metallb
-    

roles/k3s/master/templates/vip.rbac.yaml.j2

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes", "endpoints"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

roles/k3s/node/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Name of the master group
+group_name_master: master

roles/k3s/post/tasks/main.yml

@@ -1,107 +0,0 @@
----
-- name: Create manifests directory for temp configuration
-  file:
-    path: /tmp/k3s
-    state: directory
-    owner: root
-    group: root
-    mode: 0644
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Copy metallb CRs manifest to first master
-  template:
-    src: "metallb.crs.j2"
-    dest: "/tmp/k3s/metallb-crs.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Test metallb-system namespace
-  command: >-
-    k3s kubectl -n metallb-system
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for metallb controller to be running
-  command: >-
-    kubectl wait deployment -n metallb-system controller --for condition=Available=True --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for metallb webhook service to be running
-  command: >-
-    kubectl wait -n metallb-system --for=jsonpath='{.status.phase}'=Running pods \
-    --selector component=controller --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for metallb pods in replicasets
-  command: >-
-    kubectl wait pods -n metallb-system --for condition=Ready \
-    --selector component=controller,app=metallb --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for the metallb controller readyReplicas
-  command: >-
-    kubectl wait -n metallb-system --for=jsonpath='{.status.readyReplicas}'=1 replicasets \
-      --selector component=controller,app=metallb --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for the metallb controller fullyLabeledReplicas
-  command: >-
-    kubectl wait -n metallb-system --for=jsonpath='{.status.fullyLabeledReplicas}'=1 replicasets \
-      --selector component=controller,app=metallb --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Wait for the metallb  controller availableReplicas
-  command: >-
-    kubectl wait -n metallb-system --for=jsonpath='{.status.availableReplicas}'=1 replicasets \
-      --selector component=controller,app=metallb --timeout=60s
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Test metallb-system webhook-service endpoint
-  command: >-
-    k3s kubectl -n metallb-system get endpoints webhook-service
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Apply metallb CRs
-  command: >-
-    k3s kubectl apply -f /tmp/k3s/metallb-crs.yaml
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Test metallb-system IPAddressPool
-  command: >-
-    k3s kubectl -n metallb-system get IPAddressPool
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Test metallb-system L2Advertisement
-  command: >-
-    k3s kubectl -n metallb-system get L2Advertisement
-  changed_when: false
-  with_items: "{{ groups['master'] }}"
-  run_once: true
-
-- name: Remove tmp director used for manifests
-  file:
-    path: /tmp/k3s
-    state: absent

roles/k3s/post/tasks/templates/metallb.crs.j2

@@ -1,14 +0,0 @@
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: first-pool
-  namespace: metallb-system
-spec:
-  addresses:
-  - {{ metal_lb_ip_range }}
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: default
-  namespace: metallb-system

roles/k3s_agent/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s-node.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s-node.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s-node.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_agent/tasks/main.yml

@@ -1,5 +1,9 @@
 ---
 
+- name: Deploy K3s http_proxy conf
+  include_tasks: http_proxy.yml
+  when: proxy_env is defined
+
 - name: Copy K3s service file
   template:
     src: "k3s.service.j2"
@@ -11,6 +15,6 @@
 - name: Enable and check K3s service
   systemd:
     name: k3s-node
-    daemon_reload: yes
+    daemon_reload: true
     state: restarted
-    enabled: yes
+    enabled: true

roles/k3s_agent/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_agent/templates/k3s.service.j2

@@ -7,7 +7,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint }}:6443 --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
+ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead

roles/k3s_custom_registries/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether custom registries for k3s should be configured
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/k3s_custom_registries/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Create directory /etc/rancher/k3s
+  file:
+    path: "/etc/{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - rancher
+    - rancher/k3s
+
+- name: Insert registries into /etc/rancher/k3s/registries.yaml
+  blockinfile:
+    path: /etc/rancher/k3s/registries.yaml
+    block: "{{ custom_registries_yaml }}"
+    mode: '0600'
+    create: true

roles/k3s_server/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+# If you want to explicitly define an interface that ALL control nodes
+# should use to propagate the VIP, define it here. Otherwise, kube-vip
+# will determine the right interface automatically at runtime.
+kube_vip_iface: null
+
+# Name of the master group
+group_name_master: master
+
+# yamllint disable rule:line-length
+server_init_args: >-
+  {% if groups[group_name_master | default('master')] | length > 1 %}
+    {% if ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] %}
+      --cluster-init
+    {% else %}
+      --server https://{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443
+    {% endif %}
+    --token {{ k3s_token }}
+  {% endif %}
+  {{ extra_server_args | default('') }}

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -0,0 +1,28 @@
+---
+# Download logs of k3s-init.service from the nodes to localhost.
+# Note that log_destination must be set.
+
+- name: Fetch k3s-init.service logs
+  ansible.builtin.command:
+    cmd: journalctl --all --unit=k3s-init.service
+  changed_when: false
+  register: k3s_init_log
+
+- name: Create {{ log_destination }}
+  delegate_to: localhost
+  run_once: true
+  become: false
+  ansible.builtin.file:
+    path: "{{ log_destination }}"
+    state: directory
+    mode: "0755"
+
+- name: Store logs to {{ log_destination }}
+  delegate_to: localhost
+  become: false
+  ansible.builtin.template:
+    src: content.j2
+    dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
+    mode: 0644
+  vars:
+    content: "{{ k3s_init_log.stdout }}"

roles/k3s_server/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_server/tasks/kube-vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip cloud provider manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy kubevip configMap manifest to first master
+  template:
+    src: "kubevip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/kubevip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/main.yml

@@ -1,63 +1,40 @@
 ---
 
-- name: Clean previous runs of k3s-init
+- name: Stop k3s-init
   systemd:
     name: k3s-init
     state: stopped
   failed_when: false
 
-- name: Clean previous runs of k3s-init
+# k3s-init won't work if the port is already in use
+- name: Stop k3s
+  systemd:
+    name: k3s
+    state: stopped
+  failed_when: false
+
+- name: Clean previous runs of k3s-init  # noqa command-instead-of-module
+  # The systemd module does not support "reset-failed", so we need to resort to command.
   command: systemctl reset-failed k3s-init
   failed_when: false
   changed_when: false
-  args:
-    warn: false  # The ansible systemd module does not support reset-failed
 
-- name: Create manifests directory on first master
+- name: Deploy K3s http_proxy conf
-  file:
+  include_tasks: http_proxy.yml
-    path: /var/lib/rancher/k3s/server/manifests
+  when: proxy_env is defined
-    state: directory
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0])
 
-- name: Copy vip rbac manifest to first master
+- name: Deploy vip manifest
-  template:
+  include_tasks: vip.yml
-    src: "vip.rbac.yaml.j2"
-    dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0])
 
-- name: Copy vip manifest to first master
+- name: Deploy metallb manifest
-  template:
+  include_tasks: metallb.yml
-    src: "vip.yaml.j2"
+  tags: metallb
-    dest: "/var/lib/rancher/k3s/server/manifests/vip.yaml"
+  when: kube_vip_lb_ip_range is not defined
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0])
 
-# these will be copied and installed now, then tested later and apply config
+- name: Deploy kube-vip manifest
-- name: Copy metallb namespace to first master
+  include_tasks: kube-vip.yml
-  template:
+  tags: kubevip
-    src: "metallb.namespace.j2"
+  when: kube_vip_lb_ip_range is defined
-    dest: "/var/lib/rancher/k3s/server/manifests/metallb-namespace.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0])
-
-- name: Copy metallb namespace to first master
-  template:
-    src: "metallb.crds.j2"
-    dest: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0])
 
 - name: Init cluster inside the transient k3s-init service
   command:
@@ -65,27 +42,31 @@
                       -p Restart=on-failure \
                       --unit=k3s-init \
                       k3s server {{ server_init_args }}"
-    creates: "{{ systemd_dir }}/k3s.service"
+    creates: "{{ systemd_dir }}/k3s-init.service"
-  args:
-    warn: false  # The ansible systemd module does not support transient units
 
 - name: Verification
+  when: not ansible_check_mode
   block:
     - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
       command:
         cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}"
       register: nodes
-      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups['master'] | length)
+      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length)  # yamllint disable-line rule:line-length
       retries: "{{ retry_count | default(20) }}"
       delay: 10
       changed_when: false
   always:
+    - name: Save logs of k3s-init.service
+      include_tasks: fetch_k3s_init_logs.yml
+      when: log_destination
+      vars:
+        log_destination: >-
+          {{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }}
     - name: Kill the temporary service used for initialization
       systemd:
         name: k3s-init
         state: stopped
       failed_when: false
-  when: not ansible_check_mode
 
 - name: Copy K3s service file
   register: k3s_service
@@ -99,9 +80,9 @@
 - name: Enable and check K3s service
   systemd:
     name: k3s
-    daemon_reload: yes
+    daemon_reload: true
     state: restarted
-    enabled: yes
+    enabled: true
 
 - name: Wait for node-token
   wait_for:
@@ -133,25 +114,32 @@
 
 - name: Create directory .kube
   file:
-    path: ~{{ ansible_user }}/.kube
+    path: "{{ ansible_user_dir }}/.kube"
     state: directory
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: "u=rwx,g=rx,o="
 
 - name: Copy config file to user home directory
   copy:
     src: /etc/rancher/k3s/k3s.yaml
-    dest: ~{{ ansible_user }}/.kube/config
+    dest: "{{ ansible_user_dir }}/.kube/config"
-    remote_src: yes
+    remote_src: true
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: "u=rw,g=,o="
 
-- name: Configure kubectl cluster to https://{{ apiserver_endpoint }}:6443
+- name: Configure kubectl cluster to {{ endpoint_url }}
   command: >-
     k3s kubectl config set-cluster default
-      --server=https://{{ apiserver_endpoint }}:6443
+      --server={{ endpoint_url }}
-      --kubeconfig ~{{ ansible_user }}/.kube/config
+      --kubeconfig {{ ansible_user_dir }}/.kube/config
   changed_when: true
+  vars:
+    endpoint_url: >-
+      https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443
+  # Deactivated linter rules:
+  #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+  #                     would be undefined. This will not be the case during playbook execution.
+  # noqa jinja[invalid]
 
 - name: Create kubectl symlink
   file:

roles/k3s_server/tasks/metallb.yml

@@ -0,0 +1,30 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: "Download to first master: manifest for metallb-{{ metal_lb_type }}"
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/metallb/metallb/{{ metal_lb_controller_tag_version }}/config/manifests/metallb-{{ metal_lb_type }}.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Set image versions in manifest for metallb-{{ metal_lb_type }}
+  ansible.builtin.replace:
+    path: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
+    regexp: "{{ item.change | ansible.builtin.regex_escape }}"
+    replace: "{{ item.to }}"
+  with_items:
+    - change: "metallb/speaker:{{ metal_lb_controller_tag_version }}"
+      to: "metallb/speaker:{{ metal_lb_speaker_tag_version }}"
+  loop_control:
+    label: "{{ item.change }} => {{ item.to }}"
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip rbac manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip/{{ kube_vip_tag_version }}/docs/manifests/rbac.yaml"
+    dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy vip manifest to first master
+  template:
+    src: "vip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/vip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/templates/content.j2

@@ -0,0 +1,5 @@
+{#
+    This is a really simple template that just outputs the
+    value of the "content" variable.
+#}
+{{ content }}

roles/k3s_server/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_server/templates/kubevip.yaml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubevip
+  namespace: kube-system
+data:
+{% if kube_vip_lb_ip_range is string %}
+{# kube_vip_lb_ip_range was used in the legacy way: single string instead of a list #}
+{#   => transform to list with single element #}
+{% set kube_vip_lb_ip_range = [kube_vip_lb_ip_range] %}
+{% endif %}
+  range-global: {{ kube_vip_lb_ip_range | join(',') }}

roles/k3s_server/templates/vip.yaml.j2

@@ -30,10 +30,12 @@ spec:
           value: "true"
         - name: port
           value: "6443"
+{% if kube_vip_iface %}
         - name: vip_interface
-          value: {{ flannel_iface }}
+          value: {{ kube_vip_iface }}
+{% endif %}
         - name: vip_cidr
-          value: "32"
+          value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
         - name: cp_enable
           value: "true"
         - name: cp_namespace
@@ -41,7 +43,7 @@ spec:
         - name: vip_ddns
           value: "false"
         - name: svc_enable
-          value: "false"
+          value: "{{ 'true' if kube_vip_lb_ip_range is defined else 'false' }}"
         - name: vip_leaderelection
           value: "true"
         - name: vip_leaseduration

roles/k3s_server_post/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Timeout to wait for MetalLB services to come up
+metal_lb_available_timeout: 240s
+
+# Name of the master group
+group_name_master: master

roles/k3s_server_post/tasks/calico.yml

@@ -0,0 +1,114 @@
+---
+- name: Deploy Calico to cluster
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+  run_once: true
+  block:
+    - name: Create manifests directory on first master
+      file:
+        path: /tmp/k3s
+        state: directory
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml"
+        dest: "/tmp/k3s/tigera-operator.yaml"
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Copy Calico custom resources manifest to first master
+      ansible.builtin.template:
+        src: "calico.crs.j2"
+        dest: /tmp/k3s/custom-resources.yaml
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Deploy or replace Tigera Operator
+      block:
+        - name: Deploy Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/tigera-operator.yaml
+          register: create_operator
+          changed_when: "'created' in create_operator.stdout"
+          failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr"
+      rescue:
+        - name: Replace existing Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl replace -f /tmp/k3s/tigera-operator.yaml
+          register: replace_operator
+          changed_when: "'replaced' in replace_operator.stdout"
+          failed_when: "'Error' in replace_operator.stderr"
+
+    - name: Wait for Tigera Operator resources
+      command: >-
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='tigera-operator'
+        --for=condition=Available=True
+        --timeout=7s
+      register: tigera_result
+      changed_when: false
+      until: tigera_result is succeeded
+      retries: 7
+      delay: 7
+      with_items:
+        - {name: tigera-operator, type: deployment}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Deploy Calico custom resources
+      block:
+        - name: Deploy custom resources for Calico
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/custom-resources.yaml
+          register: create_cr
+          changed_when: "'created' in create_cr.stdout"
+          failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr"
+      rescue:
+        - name: Apply new Calico custom resource manifest
+          ansible.builtin.command:
+            cmd: kubectl apply -f /tmp/k3s/custom-resources.yaml
+          register: apply_cr
+          changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
+          failed_when: "'Error' in apply_cr.stderr"
+
+    - name: Wait for Calico system resources to be available
+      command: >-
+        {% if item.type == 'daemonset' %}
+        k3s kubectl wait pods
+        --namespace='{{ item.namespace }}'
+        --selector={{ item.selector }}
+        --for=condition=Ready
+        {% else %}
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='{{ item.namespace }}'
+        --for=condition=Available
+        {% endif %}
+        --timeout=7s
+      register: cr_result
+      changed_when: false
+      until: cr_result is succeeded
+      retries: 30
+      delay: 7
+      with_items:
+        - {name: calico-typha, type: deployment, namespace: calico-system}
+        - {name: calico-kube-controllers, type: deployment, namespace: calico-system}
+        - {name: csi-node-driver, type: daemonset, selector: 'k8s-app=csi-node-driver', namespace: calico-system}
+        - {name: calico-node, type: daemonset, selector: 'k8s-app=calico-node', namespace: calico-system}
+        - {name: calico-apiserver, type: deployment, namespace: calico-apiserver}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Patch Felix configuration for eBPF mode
+      ansible.builtin.command:
+        cmd: >
+          kubectl patch felixconfiguration default
+          --type='merge'
+          --patch='{"spec": {"bpfKubeProxyIptablesCleanupEnabled": false}}'
+      register: patch_result
+      changed_when: "'felixconfiguration.projectcalico.org/default patched' in patch_result.stdout"
+      failed_when: "'Error' in patch_result.stderr"
+      when: calico_ebpf

roles/k3s_server_post/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Deploy calico
+  include_tasks: calico.yml
+  tags: calico
+  when: calico_iface is defined
+
+- name: Deploy metallb pool
+  include_tasks: metallb.yml
+  tags: metallb
+  when: kube_vip_lb_ip_range is not defined
+
+- name: Remove tmp directory used for manifests
+  file:
+    path: /tmp/k3s
+    state: absent

roles/k3s_server_post/tasks/metallb.yml

@@ -0,0 +1,122 @@
+---
+- name: Create manifests directory for temp configuration
+  file:
+    path: /tmp/k3s
+    state: directory
+    owner: "{{ ansible_user_id }}"
+    mode: 0755
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+  run_once: true
+
+- name: Delete outdated metallb replicas
+  shell: |-
+    set -o pipefail
+
+    REPLICAS=$(k3s kubectl --namespace='metallb-system' get replicasets \
+      -l 'component=controller,app=metallb' \
+      -o jsonpath='{.items[0].spec.template.spec.containers[0].image}, {.items[0].metadata.name}' 2>/dev/null || true)
+    REPLICAS_SETS=$(echo ${REPLICAS} | grep -v '{{ metal_lb_controller_tag_version }}' | sed -e "s/^.*\s//g")
+    if [ -n "${REPLICAS_SETS}" ] ; then
+      for REPLICAS in "${REPLICAS_SETS}"
+      do
+        k3s kubectl --namespace='metallb-system' \
+          delete rs "${REPLICAS}"
+      done
+    fi
+  args:
+    executable: /bin/bash
+  changed_when: false
+  run_once: true
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+
+- name: Copy metallb CRs manifest to first master
+  template:
+    src: "metallb.crs.j2"
+    dest: "/tmp/k3s/metallb-crs.yaml"
+    owner: "{{ ansible_user_id }}"
+    mode: 0755
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+  run_once: true
+
+- name: Test metallb-system namespace
+  command: >-
+    k3s kubectl -n metallb-system
+  changed_when: false
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+  run_once: true
+
+- name: Wait for MetalLB resources
+  command: >-
+    k3s kubectl wait {{ item.resource }}
+    --namespace='metallb-system'
+    {% if item.name | default(False) -%}{{ item.name }}{%- endif %}
+    {% if item.selector | default(False) -%}--selector='{{ item.selector }}'{%- endif %}
+    {% if item.condition | default(False) -%}{{ item.condition }}{%- endif %}
+    --timeout='{{ metal_lb_available_timeout }}'
+  changed_when: false
+  run_once: true
+  with_items:
+    - description: controller
+      resource: deployment
+      name: controller
+      condition: --for condition=Available=True
+    - description: webhook service
+      resource: pod
+      selector: component=controller
+      condition: --for=jsonpath='{.status.phase}'=Running
+    - description: pods in replica sets
+      resource: pod
+      selector: component=controller,app=metallb
+      condition: --for condition=Ready
+    - description: ready replicas of controller
+      resource: replicaset
+      selector: component=controller,app=metallb
+      condition: --for=jsonpath='{.status.readyReplicas}'=1
+    - description: fully labeled replicas of controller
+      resource: replicaset
+      selector: component=controller,app=metallb
+      condition: --for=jsonpath='{.status.fullyLabeledReplicas}'=1
+    - description: available replicas of controller
+      resource: replicaset
+      selector: component=controller,app=metallb
+      condition: --for=jsonpath='{.status.availableReplicas}'=1
+  loop_control:
+    label: "{{ item.description }}"
+
+- name: Test metallb-system webhook-service endpoint
+  command: >-
+    k3s kubectl -n metallb-system get endpoints webhook-service
+  changed_when: false
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+  run_once: true
+
+- name: Apply metallb CRs
+  command: >-
+    k3s kubectl apply -f /tmp/k3s/metallb-crs.yaml
+    --timeout='{{ metal_lb_available_timeout }}'
+  register: this
+  changed_when: false
+  run_once: true
+  until: this.rc == 0
+  retries: 5
+
+- name: Test metallb-system resources for Layer 2 configuration
+  command: >-
+    k3s kubectl -n metallb-system get {{ item }}
+  changed_when: false
+  run_once: true
+  when: metal_lb_mode == "layer2"
+  with_items:
+    - IPAddressPool
+    - L2Advertisement
+
+- name: Test metallb-system resources for BGP configuration
+  command: >-
+    k3s kubectl -n metallb-system get {{ item }}
+  changed_when: false
+  run_once: true
+  when: metal_lb_mode == "bgp"
+  with_items:
+    - IPAddressPool
+    - BGPPeer
+    - BGPAdvertisement

roles/k3s_server_post/templates/calico.crs.j2

@@ -0,0 +1,41 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: {{ calico_blockSize | default('26') }}
+      cidr: {{ calico_cidr | default('10.52.0.0/16') }}
+      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
+      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
+      nodeSelector: {{ calico_nodeSelector | default('all()') }}
+    nodeAddressAutodetectionV4:
+      interface: {{ calico_iface  }}
+    linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}
+
+{% if calico_ebpf %}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kubernetes-services-endpoint
+  namespace: tigera-operator
+data:
+  KUBERNETES_SERVICE_HOST: '{{ apiserver_endpoint }}'
+  KUBERNETES_SERVICE_PORT: '6443'
+{% endif %}

roles/k3s_server_post/templates/metallb.crs.j2

@@ -0,0 +1,43 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: first-pool
+  namespace: metallb-system
+spec:
+  addresses:
+{% if metal_lb_ip_range is string %}
+{# metal_lb_ip_range was used in the legacy way: single string instead of a list #}
+{#   => transform to list with single element #}
+{% set metal_lb_ip_range = [metal_lb_ip_range] %}
+{% endif %}
+{% for range in metal_lb_ip_range %}
+  - {{ range }}
+{% endfor %}
+
+{% if metal_lb_mode == "layer2" %}
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default
+  namespace: metallb-system
+{% endif %}
+{% if metal_lb_mode == "bgp" %}
+---
+apiVersion: metallb.io/v1beta2
+kind: BGPPeer
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  myASN: {{ metal_lb_bgp_my_asn }}
+  peerASN: {{ metal_lb_bgp_peer_asn }}
+  peerAddress: {{ metal_lb_bgp_peer_address }}
+
+---
+apiVersion: metallb.io/v1beta1
+kind: BGPAdvertisement
+metadata:
+  name: default
+  namespace: metallb-system
+{% endif %}

roles/lxc/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reboot server
+  become: true
+  reboot:
+  listen: reboot server

roles/lxc/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Check for rc.local file
+  stat:
+    path: /etc/rc.local
+  register: rcfile
+
+- name: Create rc.local if needed
+  lineinfile:
+    path: /etc/rc.local
+    line: "#!/bin/sh -e"
+    create: true
+    insertbefore: BOF
+    mode: "u=rwx,g=rx,o=rx"
+  when: not rcfile.stat.exists
+
+- name: Write rc.local file
+  blockinfile:
+    path: /etc/rc.local
+    content: "{{ lookup('template', 'templates/rc.local.j2') }}"
+    state: present
+  notify: reboot server

roles/prereq/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+secure_path:
+  RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+  Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin'

roles/prereq/tasks/main.yml

@@ -1,27 +1,37 @@
 ---
 - name: Set same timezone on every Server
-  timezone:
+  community.general.timezone:
     name: "{{ system_timezone }}"
   when: (system_timezone is defined) and (system_timezone != "Your/Timezone")
 
 - name: Set SELinux to disabled state
-  selinux:
+  ansible.posix.selinux:
     state: disabled
   when: ansible_os_family == "RedHat"
 
 - name: Enable IPv4 forwarding
-  sysctl:
+  ansible.posix.sysctl:
     name: net.ipv4.ip_forward
     value: "1"
     state: present
-    reload: yes
+    reload: true
+  tags: sysctl
 
 - name: Enable IPv6 forwarding
-  sysctl:
+  ansible.posix.sysctl:
     name: net.ipv6.conf.all.forwarding
     value: "1"
     state: present
-    reload: yes
+    reload: true
+  tags: sysctl
+
+- name: Enable IPv6 router advertisements
+  ansible.posix.sysctl:
+    name: net.ipv6.conf.all.accept_ra
+    value: "2"
+    state: present
+    reload: true
+  tags: sysctl
 
 - name: Add br_netfilter to /etc/modules-load.d/
   copy:
@@ -31,28 +41,29 @@
   when: ansible_os_family == "RedHat"
 
 - name: Load br_netfilter
-  modprobe:
+  community.general.modprobe:
     name: br_netfilter
     state: present
   when: ansible_os_family == "RedHat"
 
 - name: Set bridge-nf-call-iptables (just to be sure)
-  sysctl:
+  ansible.posix.sysctl:
     name: "{{ item }}"
     value: "1"
     state: present
-    reload: yes
+    reload: true
   when: ansible_os_family == "RedHat"
   loop:
     - net.bridge.bridge-nf-call-iptables
     - net.bridge.bridge-nf-call-ip6tables
+  tags: sysctl
 
 - name: Add /usr/local/bin to sudo secure_path
   lineinfile:
-    line: 'Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+    line: 'Defaults    secure_path = {{ secure_path[ansible_os_family] }}'
     regexp: "Defaults(\\s)*secure_path(\\s)*="
     state: present
     insertafter: EOF
     path: /etc/sudoers
     validate: 'visudo -cf %s'
-  when: ansible_os_family == "RedHat"
+  when: ansible_os_family in [ "RedHat", "Suse" ]

roles/proxmox_lxc/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Reboot containers
+  block:
+    - name: Get container ids from filtered files
+      set_fact:
+        proxmox_lxc_filtered_ids: >-
+          {{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }}
+      listen: reboot containers
+    - name: Reboot container
+      command: "pct reboot {{ item }}"
+      loop: "{{ proxmox_lxc_filtered_ids }}"
+      changed_when: true
+      listen: reboot containers

roles/proxmox_lxc/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+- name: Check for container files that exist on this host
+  stat:
+    path: "/etc/pve/lxc/{{ item }}.conf"
+  loop: "{{ proxmox_lxc_ct_ids }}"
+  register: stat_results
+
+- name: Filter out files that do not exist
+  set_fact:
+    proxmox_lxc_filtered_files:
+      '{{ stat_results.results | rejectattr("stat.exists", "false") | map(attribute="stat.path") }}'
+
+# https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185
+- name: Ensure lxc config has the right apparmor profile
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.apparmor.profile"
+    line: "lxc.apparmor.profile: unconfined"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cgroup
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cgroup.devices.allow"
+    line: "lxc.cgroup.devices.allow: a"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cap drop
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cap.drop"
+    line: "lxc.cap.drop: "
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right mounts
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.mount.auto"
+    line: 'lxc.mount.auto: "proc:rw sys:rw"'
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers

roles/raspberrypi/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether the k3s prerequisites for Raspberry Pi should be set up
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/raspberrypi/handlers/main.yml

@@ -1,3 +1,4 @@
 ---
 - name: Reboot
   reboot:
+  listen: reboot

roles/raspberrypi/tasks/main.yml

@@ -17,21 +17,27 @@
   when:
     grep_cpuinfo_raspberrypi.rc == 0 or grep_device_tree_model_raspberrypi.rc == 0
 
-- name: Set detected_distribution to Raspbian
+- name: Set detected_distribution to Raspbian (ARM64 on Raspbian, Debian Buster/Bullseye/Bookworm)
   set_fact:
     detected_distribution: Raspbian
-  when: >
+  vars:
-    raspberry_pi|default(false) and
+    allowed_descriptions:
-    ( ansible_facts.lsb.id|default("") == "Raspbian" or
+      - "[Rr]aspbian.*"
-      ansible_facts.lsb.description|default("") is match("[Rr]aspbian.*") )
+      - "Debian.*buster"
+      - "Debian.*bullseye"
+      - "Debian.*bookworm"
+  when:
+    - ansible_facts.architecture is search("aarch64")
+    - raspberry_pi|default(false)
+    - ansible_facts.lsb.description|default("") is match(allowed_descriptions | join('|'))
 
-- name: Set detected_distribution to Raspbian (ARM64 on Debian Buster)
+- name: Set detected_distribution to Raspbian (ARM64 on Debian Bookworm)
   set_fact:
     detected_distribution: Raspbian
   when:
     - ansible_facts.architecture is search("aarch64")
     - raspberry_pi|default(false)
-    - ansible_facts.lsb.description|default("") is match("Debian.*buster")
+    - ansible_facts.lsb.description|default("") is match("Debian.*bookworm")
 
 - name: Set detected_distribution_major_version
   set_fact:
@@ -39,21 +45,16 @@
   when:
     - detected_distribution | default("") == "Raspbian"
 
-- name: Set detected_distribution to Raspbian (ARM64 on Debian Bullseye)
+- name: Execute OS related tasks on the Raspberry Pi - {{ action_ }}
-  set_fact:
-    detected_distribution: Raspbian
-  when:
-    - ansible_facts.architecture is search("aarch64")
-    - raspberry_pi|default(false)
-    - ansible_facts.lsb.description|default("") is match("Debian.*bullseye")
-
-- name: execute OS related tasks on the Raspberry Pi
   include_tasks: "{{ item }}"
   with_first_found:
-    - "prereq/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
+    - "{{ action_ }}/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
-    - "prereq/{{ detected_distribution }}.yml"
+    - "{{ action_ }}/{{ detected_distribution }}.yml"
-    - "prereq/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
+    - "{{ action_ }}/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
-    - "prereq/{{ ansible_distribution }}.yml"
+    - "{{ action_ }}/{{ ansible_distribution }}.yml"
-    - "prereq/default.yml"
+    - "{{ action_ }}/default.yml"
+  vars:
+    action_: >-
+      {% if state == "present" %}setup{% else %}teardown{% endif %}
   when:
     - raspberry_pi|default(false)

roles/raspberrypi/tasks/setup/Raspbian.yml

@@ -8,20 +8,22 @@
   notify: reboot
 
 - name: Install iptables
-  apt: name=iptables state=present
+  apt:
+    name: iptables
+    state: present
 
 - name: Flush iptables before changing to iptables-legacy
   iptables:
     flush: true
 
 - name: Changing to iptables-legacy
-  alternatives:
+  community.general.alternatives:
     path: /usr/sbin/iptables-legacy
     name: iptables
   register: ip4_legacy
 
 - name: Changing to ip6tables-legacy
-  alternatives:
+  community.general.alternatives:
     path: /usr/sbin/ip6tables-legacy
     name: ip6tables
   register: ip6_legacy

roles/raspberrypi/tasks/setup/Rocky.yml

@@ -1,8 +1,8 @@
 ---
-- name: Enable cgroup via boot commandline if not already enabled for Centos
+- name: Enable cgroup via boot commandline if not already enabled for Rocky
   lineinfile:
     path: /boot/cmdline.txt
-    backrefs: yes
+    backrefs: true
     regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
     line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
   notify: reboot

roles/raspberrypi/tasks/setup/Ubuntu.yml

@@ -2,12 +2,12 @@
 - name: Enable cgroup via boot commandline if not already enabled for Ubuntu on a Raspberry Pi
   lineinfile:
     path: /boot/firmware/cmdline.txt
-    backrefs: yes
+    backrefs: true
     regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
     line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
   notify: reboot
-  when: not ansible_check_mode
 
 - name: Install linux-modules-extra-raspi
-  apt: name=linux-modules-extra-raspi state=present
+  apt:
-  when: (raspberry_pi) and (not ansible_check_mode)
+    name: linux-modules-extra-raspi
+    state: present