k3s-ansible/.github/workflows/integration.yml

---
name: Integration Test

on:
  pull_request:
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        service_mgr: [systemd, openrc]
        inventory: [basic, notoken]
        exclude:
          - service_mgr: openrc
            inventory: notoken
    env:
      STARTING_K3S_VERSION: v1.34.3+k3s1
      UPGRADE_K3S_VERSION: v1.35.1+k3s1
      SITE_UPGRADE_K3S_VERSION: v1.35.6+k3s1
    # K3s requires privileged containers to run inside Docker and access to cgrougs.
    steps:
      - name: Remove Unnecessary Tools
        working-directory: /
        run: |
          sudo rm -rf \
            /home/linuxbrew \
            /home/packer \
            /opt/az \
            /opt/microsoft \
            /usr/lib/firefox \
            /usr/lib/google-cloud-sdk \
            /usr/local/julia* \
            /usr/local/share/boost \
            /usr/local/share/chromium \
            /usr/local/share/powershell \
            /usr/share/az* \
            /usr/share/dotnet \
            /usr/share/miniconda \
            /usr/share/swift
          df -khl
      - name: Set container OS based on service manager
        id: set-container
        run: |
          if [ "${{ matrix.service_mgr }}" == "systemd" ]; then
            echo "container_os=rancher/systemd-node:v0.0.8" >> $GITHUB_ENV
            echo "container_cmd=/usr/lib/systemd/systemd --unit=noop.target --show-status=true" >> $GITHUB_ENV
            echo "service_mgr=systemd" >> $GITHUB_ENV
          else
            echo "container_os=jrei/openrc-alpine" >> $GITHUB_ENV
            echo "service_mgr=openrc" >> $GITHUB_ENV
          fi

      - name: Checkout codebase
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

      - name: Set up Python 3.13.
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.13.x'

      - name: Install Ansible and dependencies
        run: |
          pip install ansible netaddr
          ansible-galaxy collection install -r collections/requirements.yml

      - name: Verify Inventory
        run: ansible-inventory -i tests/${{ matrix.inventory }}.yml --list

      - name: Create Docker Network
        run: docker network create k3s-ansible

      - name: Start Docker containers
        run: |
          # Start the Server node
          docker run -d --name server-node \
            --hostname server-node \
            --privileged \
            -v /sys/fs/bpf:/sys/fs/bpf \
            --memory 2048m \
            --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
            --volume=/lib/modules:/lib/modules:ro \
            --cgroupns=host \
            --network=k3s-ansible \
            ${{ env.container_os }} ${{ env.container_cmd }} 

          # Start the Agent node
          docker run -d --name agent-node \
            --hostname agent-node \
            --privileged \
            -v /sys/fs/bpf:/sys/fs/bpf \
            --memory 2048m \
            --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
            --volume=/lib/modules:/lib/modules:ro \
            --cgroupns=host \
            --network=k3s-ansible \
            ${{ env.container_os }} ${{ env.container_cmd }} 

      - name: Install node dependencies (systemd)
        if: matrix.service_mgr == 'systemd'
        run: |
          docker exec server-node zypper install -y python3-rpm
          docker exec agent-node zypper install -y python3-rpm

      - name: Install node dependencies (openrc)
        if: matrix.service_mgr == 'openrc'
        run: |
          docker exec server-node apk add curl python3
          docker exec agent-node apk add curl python3
      
      - name: Replace k3s_version in inventory
        run: |
          sed -i "s/k3s_version: .*/k3s_version: ${STARTING_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml

      - name: Run Playbook
        env:
          ANSIBLE_FORCE_COLOR: '1'
        run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

      - name: Verify K3s is running on Server
        run: docker exec server-node k3s kubectl get nodes | grep Ready

      - name: Verify the server config file mode
        if: matrix.inventory == 'basic'
        run: docker exec server-node ls -l /etc/rancher/k3s/config.yaml | grep -q '^-rw-r-----'

      - name: Verify K3s is running on Agent (systemd)
        if: matrix.service_mgr == 'systemd'
        run: docker exec agent-node systemctl status k3s-agent | grep running

      - name: Verify K3s is running on Agent (openrc)
        if: matrix.service_mgr == 'openrc'
        run: docker exec agent-node rc-service k3s-agent status | grep started

      - name: Wait for all deployments to be ready
        run: |
          for attempt in 1 2 3 4 5 6; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 6 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Modify the k3s_version in inventory for upgrade
        run: |
          sed -i "s/k3s_version: .*/k3s_version: ${UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml

      - name: Run Upgrade Playbook
        run: ansible-playbook playbooks/upgrade.yml -i tests/${{ matrix.inventory }}.yml

      - name: Verify K3s upgraded on Server
        run: docker exec server-node k3s --version | grep ${UPGRADE_K3S_VERSION}

      - name: Verify K3s upgraded on Agent
        run: docker exec agent-node k3s --version | grep ${UPGRADE_K3S_VERSION}

      # Second upgrade, this time re-running site.yml with a bumped version: it
      # must upgrade the running runtime on both the server and the agent, not
      # just replace the on-disk binary.
      - name: Modify the k3s_version in inventory for a site.yml upgrade
        run: |
          sed -i "s/k3s_version: .*/k3s_version: ${SITE_UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml

      - name: Run site Playbook to upgrade via a version bump
        run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

      # The install script replaces the on-disk binary regardless of a restart,
      # so assert the running version the kubelet reports to the API: it only
      # advances once the service was actually restarted onto the new build.
      # This is what fails if the version-bump restart regresses.
      - name: Verify the running K3s version upgraded on Server and Agent
        run: |
          for node in server-node agent-node; do
            for attempt in 1 2 3 4 5 6; do
              running=$(docker exec server-node k3s kubectl get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
              echo "$node running version: $running"
              if echo "$running" | grep -q "${SITE_UPGRADE_K3S_VERSION}"; then
                break
              fi
              if [ "$attempt" -eq 6 ]; then
                echo "$node did not reach ${SITE_UPGRADE_K3S_VERSION}"
                exit 1
              fi
              sleep 10
            done
          done

      - name: Wait for all deployments to be ready
        run: |
          for attempt in {1..10}; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 10 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Add node-label to server config
        run: |
          printf "    server_config_yaml: |\n      node-label: foo=bar\n" >> tests/${{ matrix.inventory }}.yml

      - name: Run site Playbook again to apply new server args
        run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

      - name: Wait for all deployments to be ready
        run: |
          for attempt in {1..10}; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 10 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Verify new server args applied
        run: |
          for attempt in 1 2 3; do
            echo "Attempt $attempt: checking node label"
            output=$(docker exec server-node k3s kubectl get nodes -o jsonpath='{.items[*].metadata.labels.foo}' 2>&1)
            echo "$output"
            if echo "$output" | grep -q bar; then
              exit 0
            fi
            if [ "$attempt" -lt 3 ]; then
              sleep 10
            fi
          done
          exit 1

      - name: Debug nodes/pods on failure
        if: failure()
        run: |
          echo "NODE INFO"
          docker exec server-node k3s kubectl get nodes -o wide
          echo "POD INFO"
          docker exec server-node k3s kubectl get pods -A -o wide
          echo "CONFIG FILE"
          docker exec server-node cat /etc/rancher/k3s/config.yaml
          echo "Server ENV"
          if [ $service_mgr == "systemd" ]; then
            docker exec server-node cat /etc/systemd/system/k3s.service.env
          elif [ $service_mgr == "openrc" ]; then
            docker exec server-node cat /etc/rancher/k3s/k3s.env
          fi
          echo "Agent ENV"
          if [ $service_mgr == "systemd" ]; then
            docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
          elif [ $service_mgr == "openrc" ]; then
            docker exec agent-node cat /etc/rancher/k3s/k3s-agent.env
          fi
          echo "SERVER LOGS"
          if [ $service_mgr == "systemd" ]; then
            docker exec server-node journalctl -u k3s -n 50
          elif [ $service_mgr == "openrc" ]; then
            docker exec server-node tail -n 50 /var/log/k3s.log
          fi
          echo "AGENT LOGS"
          if [ $service_mgr == "systemd" ]; then
            docker exec agent-node journalctl -u k3s -n 50
          elif [ $service_mgr == "openrc" ]; then
            docker exec agent-node tail -n 50 /var/log/k3s.log
          fi
          

      - name: Remove K3s from Server and Agent
        run: ansible-playbook playbooks/reset.yml -i tests/${{ matrix.inventory }}.yml

      - name: Stop and remove Docker containers
        run: |
          docker stop server-node && docker rm -f server-node
          docker stop agent-node && docker rm -f agent-node
          docker network rm k3s-ansible

  test-ha:
    runs-on: ubuntu-latest
    # K3s requires privileged containers to run inside Docker and access to cgrougs.
    env:
      SERVERS: server-node1 server-node2 server-node3
    steps:
      - name: Checkout codebase
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

      - name: Set up Python 3.13.
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.13.x'

      - name: Install Ansible and dependencies
        run: |
          pip install ansible netaddr
          ansible-galaxy collection install -r collections/requirements.yml

      - name: Verify Inventory
        run: ansible-inventory -i tests/ha.yml --list

      - name: Create Docker Network
        run: docker network create k3s-ha-ansible

      - name: Start Docker containers
        run: |
          for SERVER in $SERVERS; do
            docker run -d --name $SERVER \
              --hostname $SERVER \
              --privileged \
              --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
              --volume=/lib/modules:/lib/modules:ro \
              --cgroupns=host \
              --network=k3s-ha-ansible \
              rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
          done

      - name: Install server dependencies
        run: |
          for SERVER in $SERVERS; do
            docker exec $SERVER zypper install -y python3-rpm
          done

      - name: Run Playbook
        env:
          ANSIBLE_FORCE_COLOR: '1'
        run: ansible-playbook playbooks/site.yml -i tests/ha.yml

      - name: Verify K3s is running on servers
        run: docker exec server-node1 k3s kubectl get nodes | grep Ready | wc -l | grep 3

      - name: Wait for all deployments to be ready
        run: |
          for attempt in 1 2 3 4 5 6; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 6 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Modify the k3s_version in inventory for upgrade
        run: |
          sed -i 's/k3s_version: v1.33.4+k3s1/k3s_version: v1.34.1+k3s1/' tests/ha.yml

      - name: Run Upgrade Playbook
        run: ansible-playbook playbooks/upgrade.yml -i tests/ha.yml

      - name: Verify K3s upgraded on all servers
        run: |
          sleep 5 && docker exec server-node1 k3s kubectl get nodes
          for SERVER in $SERVERS; do
            docker exec $SERVER k3s --version | grep v1.34.
          done
        
      - name: Wait for all deployments to be ready
        run: |
          for attempt in 1 2 3 4 5 6; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 6 ]; then
              sleep 15
            fi
          done
          exit 1

      # Re-running site.yml with a bumped version must upgrade every etcd
      # server. k3s_server_serial=1 restarts them one at a time so the etcd
      # quorum is never lost.
      - name: Modify the k3s_version in inventory for a site.yml upgrade
        run: |
          sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml

      - name: Run site Playbook to upgrade the servers via a version bump
        run: ansible-playbook playbooks/site.yml -i tests/ha.yml -e k3s_server_serial=1

      # Assert the running (kubelet-reported) version, not the on-disk binary:
      # without a rolling restart the joined servers stay on the old runtime.
      - name: Verify the running K3s version upgraded on all servers via site.yml
        run: |
          docker exec server-node1 k3s kubectl get nodes -o wide
          for SERVER in $SERVERS; do
            for attempt in 1 2 3 4 5 6; do
              running=$(docker exec server-node1 k3s kubectl get node "$SERVER" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
              echo "$SERVER running version: $running"
              if echo "$running" | grep -q v1.34.3; then
                break
              fi
              if [ "$attempt" -eq 6 ]; then
                echo "$SERVER did not reach v1.34.3"
                exit 1
              fi
              sleep 10
            done
          done

      - name: Wait for all deployments to be ready after the site.yml upgrade
        run: |
          for attempt in {1..10}; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 10 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Debug nodes/pods on failure
        if: failure()
        run: |
          echo "NODE INFO"
          docker exec server-node1 k3s kubectl get nodes -o wide
          echo "POD INFO"
          docker exec server-node1 k3s kubectl get pods -A -o wide
          echo "CONFIG FILES"
          for SERVER in $SERVERS; do
            docker exec $SERVER cat /etc/rancher/k3s/config.yaml
          done
          echo "Server ENVS"
          for SERVER in $SERVERS; do
            docker exec $SERVER cat /etc/systemd/system/k3s.service.env
          done
          echo "SERVER LOGSS"
          docker exec server-node1 journalctl -u k3s -n 10

      - name: Remove K3s from server nodes
        run: ansible-playbook playbooks/reset.yml -i tests/ha.yml

      - name: Stop and remove Docker containers
        run: |
          for SERVER in $SERVERS; do
            docker stop $SERVER && docker rm -f $SERVER
          done
          docker network rm k3s-ha-ansible

  test-external:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout codebase
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

      - name: Set up Python 3.13.
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.13.x'

      - name: Install Ansible and dependencies
        run: |
          pip install ansible netaddr
          ansible-galaxy collection install -r collections/requirements.yml

      - name: Verify Inventory
        run: ansible-inventory -i tests/external.yml --list

      - name: Create Docker Network
        run: docker network create k3s-ext-ansible

      - name: Start Database container
        run: |
          docker run -d --name mysqlDB \
          --network=k3s-ext-ansible \
          -p 3306:3306 -e MYSQL_ROOT_PASSWORD=e2e mysql:8.4
      
      - name: Start Server container
        run: |
          docker run -d --name server-node \
            --privileged \
            --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
            --volume=/lib/modules:/lib/modules:ro \
            --cgroupns=host \
            --network=k3s-ext-ansible \
            rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

      - name: Install server dependencies
        run: docker exec server-node zypper install -y python3-rpm

      - name: Run Playbook
        env:
          ANSIBLE_FORCE_COLOR: '1'
        run: ansible-playbook playbooks/site.yml -i tests/external.yml

      - name: Verify K3s is running on servers
        run: docker exec server-node k3s kubectl get nodes | grep Ready | wc -l | grep 1

      - name: Wait for all deployments to be ready
        run: |
          for attempt in 1 2 3 4 5 6; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 6 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Stop and remove Docker containers
        run: |
          docker stop server-node && docker rm -f server-node
          docker stop mysqlDB && docker rm -f mysqlDB
          docker network rm k3s-ext-ansible

  test-airgap:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout codebase
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

      - name: Set up Python 3.13.
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.13.x'

      - name: Install Ansible and dependencies
        run: |
          pip install ansible netaddr
          ansible-galaxy collection install -r collections/requirements.yml

      - name: Verify Inventory
        run: ansible-inventory -i tests/airgap.yml --list

      - name: Create Docker Network
        run: docker network create k3s-airgp-ansible
      
      - name: Start containers
        run: |
          docker run -d --name server-node \
            --privileged \
            --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
            --volume=/lib/modules:/lib/modules:ro \
            --cgroupns=host \
            --network=k3s-airgp-ansible \
            rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

          docker run -d --name agent-node \
            --privileged \
            --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
            --volume=/lib/modules:/lib/modules:ro \
            --cgroupns=host \
            --network=k3s-airgp-ansible \
            rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

      - name: Install node dependencies
        run: |
          docker exec server-node zypper install -y python3-rpm
          docker exec agent-node zypper install -y python3-rpm

      - name: Download K3s airgap artifacts
        run: |
          mkdir -p ./playbooks/test-airgap
          wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s-airgap-images-amd64.tar.gz 
          wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s 

      - name: Run Playbook
        env:
          ANSIBLE_FORCE_COLOR: '1'
        run: ansible-playbook playbooks/site.yml -i tests/airgap.yml

      - name: Verify K3s is running on servers
        run: docker exec server-node k3s kubectl get nodes | grep Ready | wc -l | grep 2

      - name: Validate its the correct K3s version
        run: |
          docker exec server-node k3s --version | grep v1.33.8
          docker exec agent-node k3s --version | grep v1.33.8

      - name: Wait for all deployments to be ready
        run: |
          for attempt in 1 2 3 4 5 6; do
            echo "Attempt $attempt: checking deployments"
            output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
            echo "$output"
            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
              exit 0
            fi
            if [ "$attempt" -lt 6 ]; then
              sleep 15
            fi
          done
          exit 1

      - name: Debug nodes/pods on failure
        if: failure()
        run: |
          echo "NODE INFO"
          docker exec server-node k3s kubectl get nodes -o wide
          echo "POD INFO"
          docker exec server-node k3s kubectl get pods -A -o wide
          echo "CONFIG FILE"
          docker exec server-node cat /etc/rancher/k3s/config.yaml
          echo "Server ENV"
          docker exec server-node cat /etc/systemd/system/k3s.service.env
          echo "Agent ENV"
          docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
          echo "SERVER LOGS"
          docker exec server-node journalctl -u k3s -n 10

      - name: Stop and remove Docker containers
        run: |
          docker stop server-node && docker rm -f server-node
          docker stop agent-node && docker rm -f agent-node
          docker network rm k3s-airgp-ansible