Merge branch 'master' into k3s-1-26

* Add option for install behind http_proxy

* Tidy up http_proxy usage

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,5 +11,5 @@
 - [ ] Ran `site.yml` playbook
 - [ ] Ran `reset.yml` playbook
 - [ ] Did not add any unnecessary changes
-- [ ] 🚀
 - [ ] Ran pre-commit install at least once before committing
+- [ ] 🚀

.github/download-boxes.sh

@@ -5,15 +5,10 @@
 # already present on the system.
 
 set -euo pipefail
-YQ_VERSION=v4.29.2
+
-YQ_BINARY=yq_linux_amd64
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
 
-# get yq used for filtering
-sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY} -O /usr/bin/yq &&\
-    sudo chmod +x /usr/bin/yq
-
 # Read all boxes for all platforms from the "molecule.yml" files
 all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
     yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"

.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+---
+name: "CI"
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '**/README.md'
+jobs:
+  lint:
+    uses: ./.github/workflows/lint.yml
+  test:
+    uses: ./.github/workflows/test.yml
+    needs: [lint]

.github/workflows/lint.yml

@@ -1,27 +1,22 @@
 ---
 name: Linting
 on:
-  pull_request:
+  workflow_call:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - '**/README.md'
 jobs:
   pre-commit-ci:
     name: Pre-Commit
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     env:
-      PYTHON_VERSION: "3.10"
+      PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # 4.3.0
+        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -61,12 +56,12 @@ jobs:
 
   ensure-pinned-actions:
     name: Ensure SHA Pinned Actions
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@6ca5574367befbc9efdb2fa25978084159c5902d # 1.3.0
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@af2eb3226618e2494e3d9084f515ad6dcf16e229 # 2.0.1
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -1,17 +1,11 @@
 ---
 name: Test
 on:
-  pull_request:
+  workflow_call:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - '**/README.md'
 jobs:
   molecule:
     name: Molecule
-    runs-on: self-hosted
+    runs-on: macos-12
-
     strategy:
       matrix:
         scenario:
@@ -20,41 +14,14 @@ jobs:
           - single_node
       fail-fast: false
     env:
-      PYTHON_VERSION: "3.10"
+      PYTHON_VERSION: "3.11"
-      VAGRANT_DEFAULT_PROVIDER: virtualbox
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Install Virtual Box from Oracle
-        run: |
-          echo "::group::Virtual Box"
-          wget -O- https://www.virtualbox.org/download/oracle_vbox_2016.asc | sudo gpg --dearmor --yes --output /usr/share/keyrings/oracle-virtualbox-2016.gpg
-          echo "deb [arch=amd64 signed-by=/usr/share/keyrings/oracle-virtualbox-2016.gpg] https://download.virtualbox.org/virtualbox/debian $(lsb_release -cs) contrib" | sudo tee -a /etc/apt/sources.list.d/virtualbox.list
-          sudo apt update && sudo apt install -y linux-headers-generic linux-headers-5.15.0-52-generic build-essential dkms virtualbox-dkms virtualbox-6.1
-          echo "::endgroup::"
-          echo "::group::Virtual Box Test"
-          vboxmanage --version
-          sudo /sbin/vboxconfig
-          sudo modprobe vboxdrv
-          vboxmanage --version
-          echo "::endgroup::"
-
-      - name: Install Vagrant
-        run: |
-          echo "::group::Install Vagrant"
-          wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
-          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-          sudo apt update && sudo apt install -y vagrant
-          vagrant version
-          vagrant plugin list
-          vagrant plugin install vagrant-vbguest
-          vagrant plugin list
-          echo "::endgroup::"
-
       - name: Configure VirtualBox
         run: |-
           sudo mkdir -p /etc/vbox
@@ -87,7 +54,7 @@ jobs:
         run: ./.github/download-boxes.sh
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # 4.3.0
+        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -104,6 +71,7 @@ jobs:
 
       - name: Test with molecule
         run: molecule test --scenario-name ${{ matrix.scenario }}
+        timeout-minutes: 90
         env:
           ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
           ANSIBLE_SSH_RETRIES: 4

.gitignore

@@ -1 +1,3 @@
 .env/
+*.log
+ansible.cfg

.pre-commit-config.yaml

@@ -1,21 +1,35 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: f71fa2c1f9cf5cb705f73dffe4b21f7c61470ba9  # frozen: v4.4.0
     hooks:
       - id: requirements-txt-fixer
       - id: sort-simple-yaml
       - id: detect-private-key
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.28.0
+    rev: b05e028c5881819161d11cb543fd96a30c06cceb  # frozen: v1.32.0
     hooks:
       - id: yamllint
         args: [-c=.yamllint]
   - repo: https://github.com/ansible-community/ansible-lint.git
-    rev: v6.8.2
+    rev: 3293b64b939c0de16ef8cb81dd49255e475bf89a  # frozen: v6.17.2
     hooks:
       - id: ansible-lint
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.8.0.4
+    rev: 375289a39f5708101b1f916eb729e8d6da96993f  # frozen: v0.9.0.5
     hooks:
       - id: shellcheck
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: 12885e376b93dc4536ad68d156065601e4433665  # frozen: v1.5.1
+    hooks:
+      - id: remove-crlf
+      - id: remove-tabs
+  - repo: https://github.com/sirosen/texthooks
+    rev: c4ffd3e31669dd4fa4d31a23436cc13839730084  # frozen: 0.5.0
+    hooks:
+      - id: fix-smartquotes

README.md

@@ -4,13 +4,13 @@
 
 This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`.
 
-This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.chipzoller.dev/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
+This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
 
 If you want more context on how this works, see:
 
-📄 [Documentation](https://docs.technotim.live/posts/k3s-etcd-ansible/) (including example commands)
+📄 [Documentation](https://technotim.live/posts/k3s-etcd-ansible/) (including example commands)
 
-📺 [Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
+📺 [Watch the Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
 
 ## 📖 k3s Ansible Playbook
 
@@ -28,14 +28,14 @@ on processor architecture:
 
 ## ✅ System requirements
 
-- Deployment environment must have Ansible 2.4.0+.  If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://docs.technotim.live/posts/ansible-automation/).
+- Control Node (the machine you are running `ansible` commands) must have Ansible 2.11+ If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://technotim.live/posts/ansible-automation/).
+
+- You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml` (important❗)
 
 - [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
 
 - `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command.
 
-- You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml`
-
 ## 🚀 Getting Started
 
 ### 🍴 Preparation
@@ -67,6 +67,8 @@ node
 
 If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/).
 
+Finally, copy `ansible.example.cfg` to `ansible.cfg` and adapt the inventory path to match the files that you just created.
+
 This requires at least k3s version `1.19.1` however the version is configurable by using the `k3s_version` variable.
 
 If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
@@ -99,7 +101,7 @@ scp debian@master_ip:~/.kube/config ~/.kube/config
 
 ### 🔨 Testing your cluster
 
-See the commands [here](https://docs.technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
+See the commands [here](https://technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
 
 ### Troubleshooting
 
@@ -112,9 +114,13 @@ It is run automatically in CI, but you can also run the tests locally.
 This might be helpful for quick feedback in a few cases.
 You can find more information about it [here](molecule/README.md).
 
+### Pre-commit Hooks
+
+This repo uses `pre-commit` and `pre-commit-hooks` to lint and fix common style and syntax errors.  Be sure to install python packages and then run `pre-commit install`.  For more information, see [pre-commit](https://pre-commit.com/)
+
 ## Thanks 🤝
 
-This repo is really standing on the shoulders of giants. Thank you to all those who have contributed and tanks to these repos for code and ideas:
+This repo is really standing on the shoulders of giants. Thank you to all those who have contributed and thanks to these repos for code and ideas:
 
 - [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible)
 - [geerlingguy/turing-pi-cluster](https://github.com/geerlingguy/turing-pi-cluster)

ansible.cfg

@@ -1,12 +0,0 @@
-[defaults]
-nocows = True
-roles_path = ./roles
-inventory  = ./hosts.ini
-
-remote_tmp = $HOME/.ansible/tmp
-local_tmp  = $HOME/.ansible/tmp
-pipelining = True
-become = True
-host_key_checking = False
-deprecation_warnings = False
-callback_whitelist = profile_tasks

ansible.example.cfg

@@ -0,0 +1,2 @@
+[defaults]
+inventory = inventory/my-cluster/hosts.ini  ; Adapt this to the path to your inventory file

deploy.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-ansible-playbook site.yml -i inventory/my-cluster/hosts.ini
+ansible-playbook site.yml

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.24.6+k3s1
+k3s_version: v1.26.11+k3s2
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -41,11 +41,89 @@ extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.5"
+kube_vip_tag_version: "v0.5.12"
+
+# metallb type frr or native
+metal_lb_type: "native"
+
+# metallb mode layer2 or bgp
+metal_lb_mode: "layer2"
+
+# bgp options
+# metal_lb_bgp_my_asn: "64513"
+# metal_lb_bgp_peer_asn: "64512"
+# metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.6"
+metal_lb_speaker_tag_version: "v0.13.9"
-metal_lb_controller_tag_version: "v0.13.6"
+metal_lb_controller_tag_version: "v0.13.9"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
+
+# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
+# in your hosts.ini file.
+# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
+# Most notably, your containers must be privileged, and must not have nesting set to true.
+# Please note this script disables most of the security of lxc containers, with the trade off being that lxc
+# containers are significantly more resource efficent compared to full VMs.
+# Mixing and matching VMs and lxc containers is not supported, ymmv if you want to do this.
+# I would only really recommend using this if you have partiularly low powered proxmox nodes where the overhead of
+# VMs would use a significant portion of your available resources.
+proxmox_lxc_configure: false
+# the user that you would use to ssh into the host, for example if you run ssh some-user@my-proxmox-host,
+# set this value to some-user
+proxmox_lxc_ssh_user: root
+# the unique proxmox ids for all of the containers in the cluster, both worker and master nodes
+proxmox_lxc_ct_ids:
+  - 200
+  - 201
+  - 202
+  - 203
+  - 204
+
+# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache
+# (harbor / nexus / docker's official registry / etc).
+# Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub),
+# or air-gapped environments where your nodes don't have internet access after the initial setup
+# (which is still needed for downloading the k3s binary and such).
+# k3s's documentation about private registries here: https://docs.k3s.io/installation/private-registry
+custom_registries: false
+# The registries can be authenticated or anonymous, depending on your registry server configuration.
+# If they allow anonymous access, simply remove the following bit from custom_registries_yaml
+#   configs:
+#     "registry.domain.com":
+#       auth:
+#         username: yourusername
+#         password: yourpassword
+# The following is an example that pulls all images used in this playbook through your private registries.
+# It also allows you to pull your own images from your private registry, without having to use imagePullSecrets
+# in your deployments.
+# If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images,
+# you can just remove those from the mirrors: section.
+custom_registries_yaml: |
+  mirrors:
+    docker.io:
+      endpoint:
+        - "https://registry.domain.com/v2/dockerhub"
+    quay.io:
+      endpoint:
+        - "https://registry.domain.com/v2/quayio"
+    ghcr.io:
+      endpoint:
+        - "https://registry.domain.com/v2/ghcrio"
+    registry.domain.com:
+      endpoint:
+        - "https://registry.domain.com"
+
+  configs:
+    "registry.domain.com":
+      auth:
+        username: yourusername
+        password: yourpassword
+
+# Only enable and configure these if you access the internet through a proxy
+# proxy_env:
+#   HTTP_PROXY: "http://proxy.domain.local:3128"
+#   HTTPS_PROXY: "http://proxy.domain.local:3128"
+#   NO_PROXY: "*.domain.local,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"

inventory/sample/group_vars/proxmox.yml

@@ -0,0 +1,2 @@
+---
+ansible_user: '{{ proxmox_lxc_ssh_user }}'

inventory/sample/hosts.ini

@@ -7,6 +7,11 @@
 192.168.30.41
 192.168.30.42
 
+# only required if proxmox_lxc_configure: true
+# must contain all proxmox instances that have a master or worker node
+# [proxmox]
+# 192.168.30.43
+
 [k3s_cluster:children]
 master
 node

molecule/default/molecule.yml

@@ -3,56 +3,73 @@ dependency:
   name: galaxy
 driver:
   name: vagrant
-.platform_presets:
+platforms:
-  - &control
+
+  - name: control1
+    box: generic/ubuntu2204
     memory: 2048
     cpus: 2
     groups:
       - k3s_cluster
       - master
-  - &node
+    interfaces:
-    memory: 2048
+      - network_name: private_network
-    cpus: 2
+        ip: 192.168.30.38
-    groups:
-      - k3s_cluster
-      - node
-  - &debian
-    box: generic/debian11
-  - &rocky
-    box: generic/rocky9
-  - &ubuntu
-    box: generic/ubuntu2204
     config_options:
       # We currently can not use public-key based authentication on Ubuntu 22.04,
       # see: https://github.com/chef/bento/issues/1405
       ssh.username: "vagrant"
       ssh.password: "vagrant"
-platforms:
+
-  - <<: [*control, *ubuntu]
+  - name: control2
-    name: control1
+    box: generic/debian11
-    interfaces:
+    memory: 2048
-      - network_name: private_network
+    cpus: 2
-        ip: 192.168.30.38
+    groups:
-  - <<: [*control, *debian]
+      - k3s_cluster
-    name: control2
+      - master
     interfaces:
       - network_name: private_network
         ip: 192.168.30.39
-  - <<: [*control, *rocky]
+
-    name: control3
+  - name: control3
+    box: generic/rocky9
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
     interfaces:
       - network_name: private_network
         ip: 192.168.30.40
-  - <<: [*node, *ubuntu]
+
-    name: node1
+  - name: node1
+    box: generic/ubuntu2204
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
     interfaces:
       - network_name: private_network
         ip: 192.168.30.41
-  - <<: [*node, *rocky]
+    config_options:
-    name: node2
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: node2
+    box: generic/rocky9
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
     interfaces:
       - network_name: private_network
         ip: 192.168.30.42
+
 provisioner:
   name: ansible
   playbooks:

molecule/default/overrides.yml

@@ -4,7 +4,8 @@
   tasks:
     - name: Override host variables
       ansible.builtin.set_fact:
-        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:

molecule/ipv6/host_vars/control2.yml

@@ -0,0 +1,3 @@
+---
+node_ipv4: 192.168.123.12
+node_ipv6: fdad:bad:ba55::de:12

molecule/ipv6/molecule.yml

@@ -3,37 +3,54 @@ dependency:
   name: galaxy
 driver:
   name: vagrant
-.platform_presets:
+platforms:
-  - &control
+  - name: control1
+    box: generic/ubuntu2204
     memory: 2048
     cpus: 2
     groups:
       - k3s_cluster
       - master
-  - &node
+    interfaces:
-    memory: 2048
+      - network_name: private_network
-    cpus: 2
+        ip: fdad:bad:ba55::de:11
-    groups:
-      - k3s_cluster
-      - node
-  - &ubuntu
-    box: generic/ubuntu2204
     config_options:
       # We currently can not use public-key based authentication on Ubuntu 22.04,
       # see: https://github.com/chef/bento/issues/1405
       ssh.username: "vagrant"
       ssh.password: "vagrant"
-platforms:
+
-  - <<: [*control, *ubuntu]
+  - name: control2
-    name: control1
+    box: generic/ubuntu2204
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - master
     interfaces:
       - network_name: private_network
-        ip: fdad:bad:ba55::de:11
+        ip: fdad:bad:ba55::de:12
-  - <<: [*node, *ubuntu]
+    config_options:
-    name: node1
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+
+  - name: node1
+    box: generic/ubuntu2204
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
     interfaces:
       - network_name: private_network
         ip: fdad:bad:ba55::de:21
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
 provisioner:
   name: ansible
   playbooks:

molecule/ipv6/overrides.yml

@@ -4,9 +4,15 @@
   tasks:
     - name: Override host variables (1/2)
       ansible.builtin.set_fact:
-        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
 
+        # In this scenario, we have multiple interfaces that the VIP could be
+        # broadcasted on. Since we have assigned a dedicated private network
+        # here, let's make sure that it is used.
+        kube_vip_iface: eth1
+
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45
 

molecule/resources/verify.yml

@@ -2,4 +2,4 @@
 - name: Verify
   hosts: all
   roles:
-    - verify/from_outside
+    - verify_from_outside

molecule/resources/verify_from_outside/defaults/main.yml

@@ -6,4 +6,4 @@ outside_host: localhost
 testing_namespace: molecule-verify-from-outside
 
 # The directory in which the example manifests reside
-example_manifests_path: ../../../../example
+example_manifests_path: ../../../example

molecule/resources/verify_from_outside/tasks/test/deploy-example.yml

@@ -34,14 +34,14 @@
 
     - name: Assert that the nginx welcome page is available
       ansible.builtin.uri:
-        url: http://{{ ip | ansible.utils.ipwrap }}:{{ port }}/
+        url: http://{{ ip | ansible.utils.ipwrap }}:{{ port_ }}/
         return_content: yes
       register: result
       failed_when: "'Welcome to nginx!' not in result.content"
       vars:
         ip: >-
           {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
-        port: >-
+        port_: >-
           {{ nginx_services.resources[0].spec.ports[0].port }}
       # Deactivated linter rules:
       #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap

molecule/single_node/overrides.yml

@@ -4,7 +4,8 @@
   tasks:
     - name: Override host variables
       ansible.builtin.set_fact:
-        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:

reboot.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ansible-playbook reboot.yml

reboot.yml

@@ -0,0 +1,9 @@
+---
+- name: Reboot k3s_cluster
+  hosts: k3s_cluster
+  gather_facts: yes
+  tasks:
+    - name: Reboot the nodes (and Wait upto 5 mins max)
+      become: true
+      reboot:
+        reboot_timeout: 300

requirements.in

@@ -0,0 +1,10 @@
+ansible-core>=2.13.5
+jmespath>=1.0.1
+jsonpatch>=1.32
+kubernetes>=25.3.0
+molecule-vagrant>=1.0.0
+molecule>=4.0.3
+netaddr>=0.8.0
+pre-commit>=2.20.0
+pre-commit-hooks>=1.3.1
+pyyaml>=6.0

requirements.txt

@@ -1,72 +1,178 @@
-ansible-compat==2.2.1
+#
-ansible-core==2.13.5
+# This file is autogenerated by pip-compile with Python 3.11
-ansible-lint==6.8.4
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+ansible-compat==3.0.1
+    # via molecule
+ansible-core==2.15.4
+    # via
+    #   -r requirements.in
+    #   ansible-compat
 arrow==1.2.3
+    # via jinja2-time
 attrs==22.1.0
+    # via jsonschema
 binaryornot==0.4.4
-black==22.10.0
+    # via cookiecutter
-bracex==2.3.post1
 cachetools==5.2.0
-Cerberus==1.3.2
+    # via google-auth
 certifi==2022.9.24
+    # via
+    #   kubernetes
+    #   requests
 cffi==1.15.1
+    # via cryptography
+cfgv==3.3.1
+    # via pre-commit
 chardet==5.0.0
+    # via binaryornot
 charset-normalizer==2.1.1
+    # via requests
 click==8.1.3
+    # via
+    #   click-help-colors
+    #   cookiecutter
+    #   molecule
 click-help-colors==0.9.1
+    # via molecule
 commonmark==0.9.1
+    # via rich
 cookiecutter==2.1.1
-cryptography==38.0.1
+    # via molecule
+cryptography==38.0.3
+    # via ansible-core
+distlib==0.3.6
+    # via virtualenv
 distro==1.8.0
+    # via selinux
 enrich==1.2.7
+    # via molecule
 filelock==3.8.0
-google-auth==2.13.0
+    # via virtualenv
+google-auth==2.14.0
+    # via kubernetes
+identify==2.5.8
+    # via pre-commit
 idna==3.4
-importlib-resources==5.10.0
+    # via requests
-Jinja2==3.1.2
+jinja2==3.1.2
+    # via
+    #   ansible-core
+    #   cookiecutter
+    #   jinja2-time
+    #   molecule
+    #   molecule-vagrant
 jinja2-time==0.2.0
+    # via cookiecutter
 jmespath==1.0.1
-jsonpatch==1.32
+    # via -r requirements.in
+jsonpatch==1.33
+    # via -r requirements.in
 jsonpointer==2.3
-jsonschema==4.16.0
+    # via jsonpatch
-kubernetes==24.2.0
+jsonschema==4.17.0
-MarkupSafe==2.1.1
+    # via
-molecule==4.0.1
+    #   ansible-compat
+    #   molecule
+kubernetes==25.3.0
+    # via -r requirements.in
+markupsafe==2.1.1
+    # via jinja2
+molecule==4.0.4
+    # via
+    #   -r requirements.in
+    #   molecule-vagrant
 molecule-vagrant==1.0.0
-mypy-extensions==0.4.3
+    # via -r requirements.in
-netaddr==0.8.0
+netaddr==0.9.0
+    # via -r requirements.in
+nodeenv==1.7.0
+    # via pre-commit
 oauthlib==3.2.2
+    # via requests-oauthlib
 packaging==21.3
-pathspec==0.10.1
+    # via
-pkgutil-resolve-name==1.3.10
+    #   ansible-compat
+    #   ansible-core
+    #   molecule
 platformdirs==2.5.2
+    # via virtualenv
 pluggy==1.0.0
-pre-commit==2.20.0
+    # via molecule
+pre-commit==2.21.0
+    # via -r requirements.in
+pre-commit-hooks==4.5.0
+    # via -r requirements.in
 pyasn1==0.4.8
+    # via
+    #   pyasn1-modules
+    #   rsa
 pyasn1-modules==0.2.8
+    # via google-auth
 pycparser==2.21
-Pygments==2.13.0
+    # via cffi
+pygments==2.13.0
+    # via rich
 pyparsing==3.0.9
-pyrsistent==0.18.1
+    # via packaging
+pyrsistent==0.19.2
+    # via jsonschema
 python-dateutil==2.8.2
+    # via
+    #   arrow
+    #   kubernetes
 python-slugify==6.1.2
+    # via cookiecutter
 python-vagrant==1.0.0
-PyYAML==6.0
+    # via molecule-vagrant
+pyyaml==6.0.1
+    # via
+    #   -r requirements.in
+    #   ansible-compat
+    #   ansible-core
+    #   cookiecutter
+    #   kubernetes
+    #   molecule
+    #   molecule-vagrant
+    #   pre-commit
 requests==2.28.1
+    # via
+    #   cookiecutter
+    #   kubernetes
+    #   requests-oauthlib
 requests-oauthlib==1.3.1
+    # via kubernetes
 resolvelib==0.8.1
+    # via ansible-core
 rich==12.6.0
+    # via
+    #   enrich
+    #   molecule
 rsa==4.9
-ruamel.yaml==0.17.21
+    # via google-auth
-ruamel.yaml.clib==0.2.7
+ruamel-yaml==0.17.21
+    # via pre-commit-hooks
 selinux==0.2.1
+    # via molecule-vagrant
 six==1.16.0
-subprocess-tee==0.3.5
+    # via
+    #   google-auth
+    #   kubernetes
+    #   python-dateutil
+subprocess-tee==0.4.1
+    # via ansible-compat
 text-unidecode==1.3
-tomli==2.0.1
+    # via python-slugify
-typing-extensions==4.4.0
 urllib3==1.26.12
-wcmatch==8.4.1
+    # via
-websocket-client==1.4.1
+    #   kubernetes
-yamllint==1.28.0
+    #   requests
-zipp==3.10.0
+virtualenv==20.16.6
+    # via pre-commit
+websocket-client==1.4.2
+    # via kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

reset.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
+ansible-playbook reset.yml

reset.yml

@@ -1,13 +1,24 @@
 ---
-
+- name: Reset k3s cluster
-- hosts: k3s_cluster
+  hosts: k3s_cluster
   gather_facts: yes
-  become: yes
   roles:
     - role: reset
+      become: true
     - role: raspberrypi
+      become: true
       vars: {state: absent}
   post_tasks:
     - name: Reboot and wait for node to come back up
+      become: true
       reboot:
         reboot_timeout: 3600
+
+- name: Revert changes to Proxmox cluster
+  hosts: proxmox
+  gather_facts: true
+  become: yes
+  remote_user: "{{ proxmox_lxc_ssh_user }}"
+  roles:
+    - role: reset_proxmox_lxc
+      when: proxmox_lxc_configure

roles/k3s/master/defaults/main.yml

@@ -1,12 +0,0 @@
----
-ansible_user: root
-server_init_args: >-
-  {% if groups['master'] | length > 1 %}
-    {% if ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname'] %}
-      --cluster-init
-    {% else %}
-      --server https://{{ hostvars[groups['master'][0]].k3s_node_ip }}:6443
-    {% endif %}
-    --token {{ k3s_token }}
-  {% endif %}
-  {{ extra_server_args | default('') }}

roles/k3s/master/templates/metallb.namespace.j2

@@ -1,6 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: metallb-system
-  labels:
-    app: metallb

roles/k3s/master/templates/vip.rbac.yaml.j2

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes", "endpoints"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

roles/k3s/node/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Name of the master group
+group_name_master: master

roles/k3s_agent/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_agent/tasks/main.yml

@@ -1,5 +1,9 @@
 ---
 
+- name: Deploy K3s http_proxy conf
+  include_tasks: http_proxy.yml
+  when: proxy_env is defined
+
 - name: Copy K3s service file
   template:
     src: "k3s.service.j2"

roles/k3s_agent/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_agent/templates/k3s.service.j2

@@ -7,7 +7,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
+ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead

roles/k3s_custom_registries/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether custom registries for k3s should be configured
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/k3s_custom_registries/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Create directory /etc/rancher/k3s
+  file:
+    path: "/etc/{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - rancher
+    - rancher/k3s
+
+- name: Insert registries into /etc/rancher/k3s/registries.yaml
+  blockinfile:
+    path: /etc/rancher/k3s/registries.yaml
+    block: "{{ custom_registries_yaml }}"
+    mode: '0600'
+    create: true

roles/k3s_server/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+# If you want to explicitly define an interface that ALL control nodes
+# should use to propagate the VIP, define it here. Otherwise, kube-vip
+# will determine the right interface automatically at runtime.
+kube_vip_iface: null
+
+# Name of the master group
+group_name_master: master
+
+# yamllint disable rule:line-length
+server_init_args: >-
+  {% if groups[group_name_master | default('master')] | length > 1 %}
+    {% if ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] %}
+      --cluster-init
+    {% else %}
+      --server https://{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443
+    {% endif %}
+    --token {{ k3s_token }}
+  {% endif %}
+  {{ extra_server_args | default('') }}

roles/k3s_server/tasks/http_proxy.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Create k3s.service.d directory
+  file:
+    path: '{{ systemd_dir }}/k3s.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+
+- name: Copy K3s http_proxy conf file
+  template:
+    src: "http_proxy.conf.j2"
+    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    owner: root
+    group: root
+    mode: '0755'

roles/k3s_server/tasks/main.yml

@@ -1,63 +1,27 @@
 ---
 
-- name: Clean previous runs of k3s-init
+- name: Stop k3s-init
   systemd:
     name: k3s-init
     state: stopped
   failed_when: false
 
-- name: Clean previous runs of k3s-init
+- name: Clean previous runs of k3s-init  # noqa command-instead-of-module
+  # The systemd module does not support "reset-failed", so we need to resort to command.
   command: systemctl reset-failed k3s-init
   failed_when: false
   changed_when: false
-  args:
-    warn: false  # The ansible systemd module does not support reset-failed
 
-- name: Create manifests directory on first master
+- name: Deploy K3s http_proxy conf
-  file:
+  include_tasks: http_proxy.yml
-    path: /var/lib/rancher/k3s/server/manifests
+  when: proxy_env is defined
-    state: directory
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']
 
-- name: Copy vip rbac manifest to first master
+- name: Deploy vip manifest
-  template:
+  include_tasks: vip.yml
-    src: "vip.rbac.yaml.j2"
-    dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']
 
-- name: Copy vip manifest to first master
+- name: Deploy metallb manifest
-  template:
+  include_tasks: metallb.yml
-    src: "vip.yaml.j2"
+  tags: metallb
-    dest: "/var/lib/rancher/k3s/server/manifests/vip.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']
-
-# these will be copied and installed now, then tested later and apply config
-- name: Copy metallb namespace to first master
-  template:
-    src: "metallb.namespace.j2"
-    dest: "/var/lib/rancher/k3s/server/manifests/metallb-namespace.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']
-
-- name: Copy metallb namespace to first master
-  template:
-    src: "metallb.crds.j2"
-    dest: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
-    owner: root
-    group: root
-    mode: 0644
-  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']
 
 - name: Init cluster inside the transient k3s-init service
   command:
@@ -66,16 +30,15 @@
                       --unit=k3s-init \
                       k3s server {{ server_init_args }}"
     creates: "{{ systemd_dir }}/k3s.service"
-  args:
-    warn: false  # The ansible systemd module does not support transient units
 
 - name: Verification
+  when: not ansible_check_mode
   block:
     - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
       command:
         cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}"
       register: nodes
-      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups['master'] | length)
+      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length)  # yamllint disable-line rule:line-length
       retries: "{{ retry_count | default(20) }}"
       delay: 10
       changed_when: false
@@ -91,7 +54,6 @@
         name: k3s-init
         state: stopped
       failed_when: false
-  when: not ansible_check_mode
 
 - name: Copy K3s service file
   register: k3s_service
@@ -139,24 +101,24 @@
 
 - name: Create directory .kube
   file:
-    path: ~{{ ansible_user }}/.kube
+    path: "{{ ansible_user_dir }}/.kube"
     state: directory
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: "u=rwx,g=rx,o="
 
 - name: Copy config file to user home directory
   copy:
     src: /etc/rancher/k3s/k3s.yaml
-    dest: ~{{ ansible_user }}/.kube/config
+    dest: "{{ ansible_user_dir }}/.kube/config"
     remote_src: yes
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: "u=rw,g=,o="
 
 - name: Configure kubectl cluster to {{ endpoint_url }}
   command: >-
     k3s kubectl config set-cluster default
       --server={{ endpoint_url }}
-      --kubeconfig ~{{ ansible_user }}/.kube/config
+      --kubeconfig {{ ansible_user_dir }}/.kube/config
   changed_when: true
   vars:
     endpoint_url: >-

roles/k3s_server/tasks/metallb.yml

@@ -0,0 +1,30 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: "Download to first master: manifest for metallb-{{ metal_lb_type }}"
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/metallb/metallb/{{ metal_lb_controller_tag_version }}/config/manifests/metallb-{{ metal_lb_type }}.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Set image versions in manifest for metallb-{{ metal_lb_type }}
+  ansible.builtin.replace:
+    path: "/var/lib/rancher/k3s/server/manifests/metallb-crds.yaml"
+    regexp: "{{ item.change | ansible.builtin.regex_escape }}"
+    replace: "{{ item.to }}"
+  with_items:
+    - change: "metallb/speaker:{{ metal_lb_controller_tag_version }}"
+      to: "metallb/speaker:{{ metal_lb_speaker_tag_version }}"
+  loop_control:
+    label: "{{ item.change }} => {{ item.to }}"
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip rbac manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip/{{ kube_vip_tag_version }}/docs/manifests/rbac.yaml"
+    dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy vip manifest to first master
+  template:
+    src: "vip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/vip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/templates/http_proxy.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+Environment=HTTP_PROXY={{ proxy_env.HTTP_PROXY }}
+Environment=HTTPS_PROXY={{ proxy_env.HTTPS_PROXY }}
+Environment=NO_PROXY={{ proxy_env.NO_PROXY }}

roles/k3s_server/templates/vip.yaml.j2

@@ -30,8 +30,10 @@ spec:
           value: "true"
         - name: port
           value: "6443"
+{% if kube_vip_iface %}
         - name: vip_interface
-          value: {{ flannel_iface }}
+          value: {{ kube_vip_iface }}
+{% endif %}
         - name: vip_cidr
           value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
         - name: cp_enable

roles/k3s_server_post/defaults/main.yml

@@ -1,3 +1,6 @@
 ---
 # Timeout to wait for MetalLB services to come up
 metal_lb_available_timeout: 120s
+
+# Name of the master group
+group_name_master: master

roles/k3s_server_post/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Deploy metallb pool
+  include_tasks: metallb.yml
+  tags: metallb
+
+- name: Remove tmp directory used for manifests
+  file:
+    path: /tmp/k3s
+    state: absent

roles/k3s_server_post/tasks/metallb.yml

@@ -3,25 +3,25 @@
   file:
     path: /tmp/k3s
     state: directory
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: 0755
-  with_items: "{{ groups['master'] }}"
+  with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
 - name: Copy metallb CRs manifest to first master
   template:
     src: "metallb.crs.j2"
     dest: "/tmp/k3s/metallb-crs.yaml"
-    owner: "{{ ansible_user }}"
+    owner: "{{ ansible_user_id }}"
     mode: 0755
-  with_items: "{{ groups['master'] }}"
+  with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
 - name: Test metallb-system namespace
   command: >-
     k3s kubectl -n metallb-system
   changed_when: false
-  with_items: "{{ groups['master'] }}"
+  with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
 - name: Wait for MetalLB resources
@@ -66,7 +66,7 @@
   command: >-
     k3s kubectl -n metallb-system get endpoints webhook-service
   changed_when: false
-  with_items: "{{ groups['master'] }}"
+  with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
 - name: Apply metallb CRs
@@ -79,16 +79,23 @@
   until: this.rc == 0
   retries: 5
 
-- name: Test metallb-system resources
+- name: Test metallb-system resources for Layer 2 configuration
   command: >-
     k3s kubectl -n metallb-system get {{ item }}
   changed_when: false
   run_once: true
+  when: metal_lb_mode == "layer2"
   with_items:
     - IPAddressPool
     - L2Advertisement
 
-- name: Remove tmp directory used for manifests
+- name: Test metallb-system resources for BGP configuration
-  file:
+  command: >-
-    path: /tmp/k3s
+    k3s kubectl -n metallb-system get {{ item }}
-    state: absent
+  changed_when: false
+  run_once: true
+  when: metal_lb_mode == "bgp"
+  with_items:
+    - IPAddressPool
+    - BGPPeer
+    - BGPAdvertisement

roles/k3s_server_post/templates/metallb.crs.j2

@@ -13,9 +13,31 @@ spec:
 {% for range in metal_lb_ip_range %}
   - {{ range }}
 {% endfor %}
+
+{% if metal_lb_mode == "layer2" %}
 ---
 apiVersion: metallb.io/v1beta1
 kind: L2Advertisement
 metadata:
   name: default
   namespace: metallb-system
+{% endif %}
+{% if metal_lb_mode == "bgp" %}
+---
+apiVersion: metallb.io/v1beta2
+kind: BGPPeer
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  myASN: {{ metal_lb_bgp_my_asn }}
+  peerASN: {{ metal_lb_bgp_peer_asn }}
+  peerAddress: {{ metal_lb_bgp_peer_address }}
+
+---
+apiVersion: metallb.io/v1beta1
+kind: BGPAdvertisement
+metadata:
+  name: default
+  namespace: metallb-system
+{% endif %}

roles/lxc/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reboot server
+  become: true
+  reboot:
+  listen: reboot server

roles/lxc/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Check for rc.local file
+  stat:
+    path: /etc/rc.local
+  register: rcfile
+
+- name: Create rc.local if needed
+  lineinfile:
+    path: /etc/rc.local
+    line: "#!/bin/sh -e"
+    create: true
+    insertbefore: BOF
+    mode: "u=rwx,g=rx,o=rx"
+  when: not rcfile.stat.exists
+
+- name: Write rc.local file
+  blockinfile:
+    path: /etc/rc.local
+    content: "{{ lookup('template', 'templates/rc.local.j2') }}"
+    state: present
+  notify: reboot server

roles/prereq/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+secure_path:
+  RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+  Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin'

roles/prereq/tasks/main.yml

@@ -1,34 +1,37 @@
 ---
 - name: Set same timezone on every Server
-  timezone:
+  community.general.timezone:
     name: "{{ system_timezone }}"
   when: (system_timezone is defined) and (system_timezone != "Your/Timezone")
 
 - name: Set SELinux to disabled state
-  selinux:
+  ansible.posix.selinux:
     state: disabled
   when: ansible_os_family == "RedHat"
 
 - name: Enable IPv4 forwarding
-  sysctl:
+  ansible.posix.sysctl:
     name: net.ipv4.ip_forward
     value: "1"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Enable IPv6 forwarding
-  sysctl:
+  ansible.posix.sysctl:
     name: net.ipv6.conf.all.forwarding
     value: "1"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Enable IPv6 router advertisements
-  sysctl:
+  ansible.posix.sysctl:
     name: net.ipv6.conf.all.accept_ra
     value: "2"
     state: present
     reload: yes
+  tags: sysctl
 
 - name: Add br_netfilter to /etc/modules-load.d/
   copy:
@@ -38,13 +41,13 @@
   when: ansible_os_family == "RedHat"
 
 - name: Load br_netfilter
-  modprobe:
+  community.general.modprobe:
     name: br_netfilter
     state: present
   when: ansible_os_family == "RedHat"
 
 - name: Set bridge-nf-call-iptables (just to be sure)
-  sysctl:
+  ansible.posix.sysctl:
     name: "{{ item }}"
     value: "1"
     state: present
@@ -53,13 +56,14 @@
   loop:
     - net.bridge.bridge-nf-call-iptables
     - net.bridge.bridge-nf-call-ip6tables
+  tags: sysctl
 
 - name: Add /usr/local/bin to sudo secure_path
   lineinfile:
-    line: 'Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+    line: 'Defaults    secure_path = {{ secure_path[ansible_os_family] }}'
     regexp: "Defaults(\\s)*secure_path(\\s)*="
     state: present
     insertafter: EOF
     path: /etc/sudoers
     validate: 'visudo -cf %s'
-  when: ansible_os_family == "RedHat"
+  when: ansible_os_family in [ "RedHat", "Suse" ]

roles/proxmox_lxc/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Reboot containers
+  block:
+    - name: Get container ids from filtered files
+      set_fact:
+        proxmox_lxc_filtered_ids: >-
+          {{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }}
+      listen: reboot containers
+    - name: Reboot container
+      command: "pct reboot {{ item }}"
+      loop: "{{ proxmox_lxc_filtered_ids }}"
+      changed_when: true
+      listen: reboot containers

roles/proxmox_lxc/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+- name: Check for container files that exist on this host
+  stat:
+    path: "/etc/pve/lxc/{{ item }}.conf"
+  loop: "{{ proxmox_lxc_ct_ids }}"
+  register: stat_results
+
+- name: Filter out files that do not exist
+  set_fact:
+    proxmox_lxc_filtered_files:
+      '{{ stat_results.results | rejectattr("stat.exists", "false") | map(attribute="stat.path") }}'
+
+# https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185
+- name: Ensure lxc config has the right apparmor profile
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.apparmor.profile"
+    line: "lxc.apparmor.profile: unconfined"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cgroup
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cgroup.devices.allow"
+    line: "lxc.cgroup.devices.allow: a"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cap drop
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cap.drop"
+    line: "lxc.cap.drop: "
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right mounts
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.mount.auto"
+    line: 'lxc.mount.auto: "proc:rw sys:rw"'
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers

roles/raspberrypi/handlers/main.yml

@@ -1,3 +1,4 @@
 ---
 - name: Reboot
   reboot:
+  listen: reboot

roles/raspberrypi/tasks/main.yml

@@ -47,20 +47,16 @@
     - raspberry_pi|default(false)
     - ansible_facts.lsb.description|default("") is match("Debian.*bullseye")
 
-- name: execute OS related tasks on the Raspberry Pi - {{ action }}
+- name: Execute OS related tasks on the Raspberry Pi - {{ action_ }}
   include_tasks: "{{ item }}"
   with_first_found:
-    - "{{ action }}/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
+    - "{{ action_ }}/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
-    - "{{ action }}/{{ detected_distribution }}.yml"
+    - "{{ action_ }}/{{ detected_distribution }}.yml"
-    - "{{ action }}/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
+    - "{{ action_ }}/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
-    - "{{ action }}/{{ ansible_distribution }}.yml"
+    - "{{ action_ }}/{{ ansible_distribution }}.yml"
-    - "{{ action }}/default.yml"
+    - "{{ action_ }}/default.yml"
   vars:
-    action: >-
+    action_: >-
-      {% if state == "present" -%}
+      {% if state == "present" %}setup{% else %}teardown{% endif %}
-      setup
-      {%- else -%}
-      teardown
-      {%- endif %}
   when:
     - raspberry_pi|default(false)

roles/raspberrypi/tasks/setup/Raspbian.yml

@@ -8,20 +8,22 @@
   notify: reboot
 
 - name: Install iptables
-  apt: name=iptables state=present
+  apt:
+    name: iptables
+    state: present
 
 - name: Flush iptables before changing to iptables-legacy
   iptables:
     flush: true
 
 - name: Changing to iptables-legacy
-  alternatives:
+  community.general.alternatives:
     path: /usr/sbin/iptables-legacy
     name: iptables
   register: ip4_legacy
 
 - name: Changing to ip6tables-legacy
-  alternatives:
+  community.general.alternatives:
     path: /usr/sbin/ip6tables-legacy
     name: ip6tables
   register: ip6_legacy

roles/reset/tasks/main.yml

@@ -46,6 +46,15 @@
     - /var/lib/rancher/
     - /var/lib/cni/
 
+- name: Remove K3s http_proxy files
+  file:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ systemd_dir }}/k3s.service.d"
+    - "{{ systemd_dir }}/k3s-node.service.d"
+  when: proxy_env is defined
+
 - name: Reload daemon_reload
   systemd:
     daemon_reload: yes
@@ -54,3 +63,31 @@
   file:
     path: /tmp/k3s
     state: absent
+
+- name: Check if rc.local exists
+  stat:
+    path: /etc/rc.local
+  register: rcfile
+
+- name: Remove rc.local modifications for proxmox lxc containers
+  become: true
+  blockinfile:
+    path: /etc/rc.local
+    content: "{{ lookup('template', 'templates/rc.local.j2') }}"
+    create: false
+    state: absent
+  when: proxmox_lxc_configure and rcfile.stat.exists
+
+- name: Check rc.local for cleanup
+  become: true
+  slurp:
+    src: /etc/rc.local
+  register: rcslurp
+  when: proxmox_lxc_configure and rcfile.stat.exists
+
+- name: Cleanup rc.local if we only have a Shebang line
+  become: true
+  file:
+    path: /etc/rc.local
+    state: absent
+  when: proxmox_lxc_configure and rcfile.stat.exists and ((rcslurp.content | b64decode).splitlines() | length) <= 1

roles/reset/tasks/umount_with_children.yml

@@ -9,7 +9,7 @@
   check_mode: false
 
 - name: Umount filesystem
-  mount:
+  ansible.posix.mount:
     path: "{{ item }}"
     state: unmounted
   with_items:

roles/reset_proxmox_lxc/handlers/main.yml

@@ -0,0 +1 @@
+../../proxmox_lxc/handlers/main.yml

roles/reset_proxmox_lxc/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+- name: Check for container files that exist on this host
+  stat:
+    path: "/etc/pve/lxc/{{ item }}.conf"
+  loop: "{{ proxmox_lxc_ct_ids }}"
+  register: stat_results
+
+- name: Filter out files that do not exist
+  set_fact:
+    proxmox_lxc_filtered_files:
+      '{{ stat_results.results | rejectattr("stat.exists", "false") | map(attribute="stat.path") }}'
+
+- name: Remove LXC apparmor profile
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.apparmor.profile"
+    line: "lxc.apparmor.profile: unconfined"
+    state: absent
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Remove lxc cgroups
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cgroup.devices.allow"
+    line: "lxc.cgroup.devices.allow: a"
+    state: absent
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Remove lxc cap drop
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cap.drop"
+    line: "lxc.cap.drop: "
+    state: absent
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Remove lxc mounts
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.mount.auto"
+    line: 'lxc.mount.auto: "proc:rw sys:rw"'
+    state: absent
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers

site.yml

@@ -1,24 +1,48 @@
 ---
+- name: Prepare Proxmox cluster
+  hosts: proxmox
+  gather_facts: true
+  become: yes
+  environment: "{{ proxy_env | default({}) }}"
+  roles:
+    - role: proxmox_lxc
+      when: proxmox_lxc_configure
 
-- hosts: k3s_cluster
+- name: Prepare k3s nodes
+  hosts: k3s_cluster
   gather_facts: yes
-  become: yes
+  environment: "{{ proxy_env | default({}) }}"
   roles:
+    - role: lxc
+      become: true
+      when: proxmox_lxc_configure
     - role: prereq
+      become: true
     - role: download
+      become: true
     - role: raspberrypi
+      become: true
+    - role: k3s_custom_registries
+      become: true
+      when: custom_registries
 
-- hosts: master
+- name: Setup k3s servers
-  become: yes
+  hosts: master
+  environment: "{{ proxy_env | default({}) }}"
   roles:
-    - role: k3s/master
+    - role: k3s_server
+      become: true
 
-- hosts: node
+- name: Setup k3s agents
-  become: yes
+  hosts: node
+  environment: "{{ proxy_env | default({}) }}"
   roles:
-    - role: k3s/node
+    - role: k3s_agent
+      become: true
 
-- hosts: master
+- name: Configure k3s cluster
-  become: yes
+  hosts: master
+  environment: "{{ proxy_env | default({}) }}"
   roles:
-    - role: k3s/post
+    - role: k3s_server_post
+      become: true

templates/rc.local.j2

@@ -0,0 +1,8 @@
+# Kubeadm 1.15 needs /dev/kmsg to be there, but it's not in lxc, but we can just use /dev/console instead
+# see: https://github.com/kubernetes-sigs/kind/issues/662
+if [ ! -e /dev/kmsg ]; then
+    ln -s /dev/console /dev/kmsg
+fi
+
+# https://medium.com/@kvaps/run-kubernetes-in-lxc-container-f04aa94b6c9c
+mount --make-rshared /