mirror of
https://github.com/k3s-io/k3s-ansible.git
synced 2026-03-09 12:02:11 +01:00
5299941270
- Use rancher/systemd-node which doesn't cause k3s to crash - Expand requirements for testing, ensure deployments become ready at each playbook - Add logging on failure for debugging Signed-off-by: Derek Nola <derek.nola@suse.com>
333 lines
12 KiB
YAML
333 lines
12 KiB
YAML
---
|
|
name: Integration Test
|
|
|
|
on:
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
service_mgr: [systemd, openrc]
|
|
|
|
# K3s requires privileged containers to run inside Docker and access to cgrougs.
|
|
steps:
|
|
- name: Set container OS based on service manager
|
|
id: set-container
|
|
run: |
|
|
if [ "${{ matrix.service_mgr }}" == "systemd" ]; then
|
|
echo "container_os=rancher/systemd-node:v0.0.8" >> $GITHUB_ENV
|
|
echo "container_cmd=/usr/lib/systemd/systemd --unit=noop.target --show-status=true" >> $GITHUB_ENV
|
|
echo "service_mgr=systemd" >> $GITHUB_ENV
|
|
else
|
|
echo "container_os=jrei/openrc-alpine" >> $GITHUB_ENV
|
|
echo "service_mgr=openrc" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/basic.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-ansible
|
|
|
|
- name: Start Docker containers
|
|
run: |
|
|
# Start the Server node
|
|
docker run -d --name server-node \
|
|
--hostname server-node \
|
|
--privileged \
|
|
-v /sys/fs/bpf:/sys/fs/bpf \
|
|
--memory 2048m \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
|
|
# Start the Agent node
|
|
docker run -d --name agent-node \
|
|
--hostname agent-node \
|
|
--privileged \
|
|
-v /sys/fs/bpf:/sys/fs/bpf \
|
|
--memory 2048m \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ansible \
|
|
${{ env.container_os }} ${{ env.container_cmd }}
|
|
|
|
- name: Install server dependencies
|
|
run: docker exec server-node zypper install -y python3-rpm
|
|
|
|
- name: Install agent dependencies (systemd)
|
|
if: matrix.service_mgr == 'systemd'
|
|
run: docker exec agent-node zypper install -y python3-rpm
|
|
|
|
- name: Install agent dependencies (openrc)
|
|
if: matrix.service_mgr == 'openrc'
|
|
run: docker exec agent-node apk add curl python3
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/basic.yml
|
|
|
|
- name: Verify K3s is running on Server
|
|
run: docker exec server-node k3s kubectl get nodes | grep Ready
|
|
|
|
- name: Verify K3s is running on Agent (systemd)
|
|
if: matrix.service_mgr == 'systemd'
|
|
run: docker exec agent-node systemctl status k3s-agent | grep running
|
|
|
|
- name: Verify K3s is running on Agent (openrc)
|
|
if: matrix.service_mgr == 'openrc'
|
|
run: docker exec agent-node rc-service k3s-agent status | grep started
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Modify the k3s_version in inventory for upgrade
|
|
run: |
|
|
sed -i 's/k3s_version: v1.33.4+k3s1/k3s_version: v1.34.1+k3s1/' tests/basic.yml
|
|
|
|
- name: Run Upgrade Playbook
|
|
run: ansible-playbook playbooks/upgrade.yml -i tests/basic.yml
|
|
|
|
- name: Verify K3s upgraded on Server
|
|
run: docker exec server-node k3s --version | grep v1.34.
|
|
|
|
- name: Verify K3s upgraded on Agent
|
|
run: docker exec agent-node k3s --version | grep v1.34.
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Add node-label to server config
|
|
run: |
|
|
printf " server_config_yaml: |\n node-label: foo=bar\n" >> tests/basic.yml
|
|
|
|
- name: Run site Playbook again to apply new server args
|
|
run: ansible-playbook playbooks/site.yml -i tests/basic.yml
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Verify new server args applied
|
|
run: |
|
|
for attempt in 1 2 3; do
|
|
echo "Attempt $attempt: checking node label"
|
|
output=$(docker exec server-node k3s kubectl get nodes -o jsonpath='{.items[*].metadata.labels.foo}' 2>&1)
|
|
echo "$output"
|
|
if echo "$output" | grep -q bar; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 3 ]; then
|
|
sleep 10
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Debug nodes/pods on failure
|
|
if: failure()
|
|
run: |
|
|
echo "NODE INFO"
|
|
docker exec server-node k3s kubectl get nodes -o wide
|
|
echo "POD INFO"
|
|
docker exec server-node k3s kubectl get pods -A -o wide
|
|
echo "CONFIG FILE"
|
|
docker exec server-node cat /etc/rancher/k3s/config.yaml
|
|
echo "Server ENV"
|
|
docker exec server-node cat /etc/systemd/system/k3s.service.env
|
|
echo "Agent ENV"
|
|
if [ $service_mgr == "systemd" ]; then
|
|
docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
|
|
elif [ $service_mgr == "openrc" ]; then
|
|
docker exec agent-node cat /etc/rancher/k3s/k3s-agent.env
|
|
fi
|
|
echo "SERVER LOGS"
|
|
docker exec server-node journalctl -u k3s -n 10
|
|
|
|
- name: Remove K3s from Server and Agent
|
|
run: ansible-playbook playbooks/reset.yml -i tests/basic.yml
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
docker stop server-node && docker rm -f server-node
|
|
docker stop agent-node && docker rm -f agent-node
|
|
docker network rm k3s-ansible
|
|
|
|
test-ha:
|
|
runs-on: ubuntu-latest
|
|
# K3s requires privileged containers to run inside Docker and access to cgrougs.
|
|
env:
|
|
SERVERS: server-node1 server-node2 server-node3
|
|
steps:
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/ha.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-ha-ansible
|
|
|
|
- name: Start Docker containers
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker run -d --name $SERVER \
|
|
--hostname $SERVER \
|
|
--privileged \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ha-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
done
|
|
|
|
- name: Install server dependencies
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER zypper install -y python3-rpm
|
|
done
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/ha.yml
|
|
|
|
- name: Verify K3s is running on servers
|
|
run: docker exec server-node1 k3s kubectl get nodes | grep Ready | wc -l | grep 3
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Modify the k3s_version in inventory for upgrade
|
|
run: |
|
|
sed -i 's/k3s_version: v1.33.4+k3s1/k3s_version: v1.34.1+k3s1/' tests/ha.yml
|
|
|
|
- name: Run Upgrade Playbook
|
|
run: ansible-playbook playbooks/upgrade.yml -i tests/ha.yml
|
|
|
|
- name: Verify K3s upgraded on all servers
|
|
run: |
|
|
sleep 5 && docker exec server-node1 k3s kubectl get nodes
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER k3s --version | grep v1.34.
|
|
done
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Debug nodes/pods on failure
|
|
if: failure()
|
|
run: |
|
|
echo "NODE INFO"
|
|
docker exec server-node1 k3s kubectl get nodes -o wide
|
|
echo "POD INFO"
|
|
docker exec server-node1 k3s kubectl get pods -A -o wide
|
|
echo "CONFIG FILES"
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER cat /etc/rancher/k3s/config.yaml
|
|
done
|
|
echo "Server ENVS"
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER cat /etc/systemd/system/k3s.service.env
|
|
done
|
|
echo "SERVER LOGSS"
|
|
docker exec server-node1 journalctl -u k3s -n 10
|
|
|
|
- name: Remove K3s from server nodes
|
|
run: ansible-playbook playbooks/reset.yml -i tests/ha.yml
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker stop $SERVER && docker rm -f $SERVER
|
|
done
|
|
docker network rm k3s-ha-ansible
|