Test playbook using molecule (#67)

* Test cluster using molecule

* Fix detection of first control node

* Include --flannel-iface and --node-ip as k3s arguments

* Store logs of k3s-init.service as GitHub job artifacts

molecule/README.md

@@ -0,0 +1,67 @@
+# Test suites for `k3s-ansible`
+
+This folder contains the [molecule](https://molecule.rtfd.io/)-based test setup for this playbook.
+
+## Scenarios
+
+We have these scenarios:
+
+- **default**:
+  A 3 control + 2 worker node cluster based very closely on the [sample inventory](../inventory/sample/).
+
+## How to execute
+
+To test on your local machine, follow these steps:
+
+### System requirements
+
+Make sure that the following software packages are available on your system:
+
+- [Python 3](https://www.python.org/downloads)
+- [Vagrant](https://www.vagrantup.com/downloads)
+- [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+
+### Set up VirtualBox networking on Linux and macOS
+
+_You can safely skip this if you are working on Windows._
+
+Furthermore, the test cluster uses the `192.168.30.0/24` subnet which is [not set up by VirtualBox automatically](https://www.virtualbox.org/manual/ch06.html#network_hostonly).
+To set the subnet up for use with VirtualBox, please make sure that `/etc/vbox/networks.conf` exists and that it contains this line:
+
+```
+* 192.168.30.0/24`
+```
+
+### Install Python dependencies
+
+You will get [Molecule, Ansible and a few extra dependencies](../requirements.txt) via [pip](https://pip.pypa.io/).
+Usually, it is advisable to work in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for this:
+
+```bash
+cd /path/to/k3s-ansible
+
+# Create a virtualenv at ".env". You only need to do this once.
+python3 -m venv .env
+
+# Activate the virtualenv for your current shell session.
+# If you start a new session, you will have to repeat this.
+source .env/bin/activate
+
+# Install the required packages into the virtualenv.
+# These remain installed across shell sessions.
+python3 -m pip install -r requirements.txt
+```
+
+### Run molecule
+
+With the virtual environment from the previous step active in your shell session, you can now use molecule to test the playbook.
+Interesting commands are:
+
+- `molecule create`: Create virtual machines for the test cluster nodes.
+- `molecule destroy`: Delete the virtual machines for the test cluster nodes.
+- `molecule converge`: Run the `site` playbook on the nodes of the test cluster.
+- `molecule side_effect`: Run the `reset` playbook on the nodes of the test cluster.
+- `molecule verify`: Verify that the cluster works correctly.
+- `molecule test`: The "all-in-one" sequence of steps that is executed in CI.
+  This includes the `create`, `converge`, `verify`, `side_effect` and `destroy` steps.
+  See [`molecule.yml`](default/molecule.yml) for more details.

molecule/default/molecule.yml

@@ -0,0 +1,75 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - &control
+    name: control1
+    box: generic/ubuntu2204
+    memory: 2048
+    cpus: 2
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.38
+  - <<: *control
+    name: control2
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.39
+  - <<: *control
+    name: control3
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.40
+  - &node
+    <<: *control
+    name: node1
+    groups:
+      - k3s_cluster
+      - node
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.41
+  - <<: *node
+    name: node2
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.42
+provisioner:
+  name: ansible
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+  env:
+    OVERRIDES_FILE: ../default/overrides.yml
+scenario:
+  test_sequence:
+    - dependency
+    - lint
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/default/overrides.yml

@@ -0,0 +1,11 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45

molecule/resources/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "OVERRIDES_FILE") }}
+
+- name: Converge
+  ansible.builtin.import_playbook: ../../site.yml

molecule/resources/reset.yml

@@ -0,0 +1,7 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "OVERRIDES_FILE") }}
+
+- name: Reset
+  ansible.builtin.import_playbook: ../../reset.yml

molecule/resources/verify.yml

@@ -0,0 +1,5 @@
+---
+- name: Verify
+  hosts: all
+  roles:
+    - verify/from_outside

molecule/resources/verify/from_outside/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# A host outside of the cluster from which the checks shall be performed
+outside_host: localhost
+
+# This kubernetes namespace will be used for testing
+testing_namespace: molecule-verify-from-outside
+
+# The directory in which the example manifests reside
+example_manifests_path: ../../../../example

molecule/resources/verify/from_outside/tasks/kubecfg-cleanup.yml

@@ -0,0 +1,5 @@
+---
+- name: Clean up kubecfg
+  ansible.builtin.file:
+    path: "{{ kubecfg.path }}"
+    state: absent

molecule/resources/verify/from_outside/tasks/kubecfg-fetch.yml

@@ -0,0 +1,19 @@
+---
+- name: Create temporary directory for kubecfg
+  ansible.builtin.tempfile:
+    state: directory
+    suffix: kubecfg
+  register: kubecfg
+- name: Gathering facts
+  delegate_to: "{{ groups['master'][0] }}"
+  ansible.builtin.gather_facts:
+- name: Download kubecfg
+  ansible.builtin.fetch:
+    src: "{{ ansible_env.HOME }}/.kube/config"
+    dest: "{{ kubecfg.path }}/"
+    flat: true
+  delegate_to: "{{ groups['master'][0] }}"
+  delegate_facts: true
+- name: Store path to kubecfg
+  ansible.builtin.set_fact:
+    kubecfg_path: "{{ kubecfg.path }}/config"

molecule/resources/verify/from_outside/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Verify
+  run_once: true
+  delegate_to: "{{ outside_host }}"
+  block:
+    - ansible.builtin.import_tasks: kubecfg-fetch.yml
+    - name: "TEST CASE: Get nodes"
+      ansible.builtin.include_tasks: test/get-nodes.yml
+    - name: "TEST CASE: Deploy example"
+      ansible.builtin.include_tasks: test/deploy-example.yml
+  always:
+    - ansible.builtin.import_tasks: kubecfg-cleanup.yml

molecule/resources/verify/from_outside/tasks/test/deploy-example.yml

@@ -0,0 +1,54 @@
+---
+- name: Deploy example
+  block:
+    - name: "Create namespace: {{ testing_namespace }}"
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: "{{ testing_namespace }}"
+        state: present
+        wait: true
+        kubeconfig: "{{ kubecfg_path }}"
+
+    - name: Apply example manifests
+      kubernetes.core.k8s:
+        src: "{{ example_manifests_path }}/{{ item }}"
+        namespace: "{{ testing_namespace }}"
+        state: present
+        wait: true
+        kubeconfig: "{{ kubecfg_path }}"
+      with_items:
+        - deployment.yml
+        - service.yml
+
+    - name: Get info about nginx service
+      kubernetes.core.k8s_info:
+        kind: service
+        name: nginx
+        namespace: "{{ testing_namespace }}"
+        kubeconfig: "{{ kubecfg_path }}"
+      vars: &load_balancer_metadata
+        metallb_ip: status.loadBalancer.ingress[0].ip
+        metallb_port: spec.ports[0].port
+      register: nginx_services
+
+    - name: Assert that the nginx welcome page is available
+      ansible.builtin.uri:
+        url: http://{{ ip }}:{{ port }}/
+        return_content: yes
+      register: result
+      failed_when: "'Welcome to nginx!' not in result.content"
+      vars:
+        ip: >-
+          {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
+        port: >-
+          {{ nginx_services.resources[0].spec.ports[0].port }}
+
+  always:
+    - name: "Remove namespace: {{ testing_namespace }}"
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: "{{ testing_namespace }}"
+        state: absent
+        kubeconfig: "{{ kubecfg_path }}"

molecule/resources/verify/from_outside/tasks/test/get-nodes.yml

@@ -0,0 +1,17 @@
+---
+- name: Get all nodes in cluster
+  kubernetes.core.k8s_info:
+    kind: node
+    kubeconfig: "{{ kubecfg_path }}"
+  register: cluster_nodes
+
+- name: Assert that the cluster contains exactly the expected nodes
+  ansible.builtin.assert:
+    that: found_nodes == expected_nodes
+    success_msg: "Found nodes as expected: {{ found_nodes }}"
+    fail_msg: "Expected nodes {{ expected_nodes }}, but found nodes {{ found_nodes }}"
+  vars:
+    found_nodes: >-
+      {{ cluster_nodes | json_query('resources[*].metadata.name') | unique }}
+    expected_nodes: >-
+      {{ (groups['master'] + groups['node']) | unique }}