mirror of
https://github.com/k3s-io/k3s-ansible.git
synced 2026-10-23 03:22:03 +02:00
* feat: make the server config file mode configurable /etc/rancher/k3s/config.yaml holds the cluster token. The server and upgrade roles write it at 0644, so every local account on a server node can read the credential that joins a node to the cluster. k3s_server_config_mode sets the mode of that file in both roles. It defaults to 0644, so behaviour is unchanged unless it is set. Signed-off-by: Kristofer <kristofer@atlascloud.is> * docs: describe the server config file contents accurately /etc/rancher/k3s/config.yaml does not always carry the cluster token. A first server that lets K3s generate one writes the file without it, so the file can contain the token rather than always holding it. Correct that claim in the README, the inventory sample and the role default, and use the US spelling of "honors". Signed-off-by: Kristofer <kristofer@atlascloud.is> * Name the floor the config file mode can be narrowed to Saying any mode root can read works leaves the reader to work out what that means in practice. The file is read only by k3s, running as root, so 0600 is the tightest mode that still works. Signed-off-by: Kristofer <kristofer@atlascloud.is> * Run the basic integration job with a non-default config file mode The templated mode was only ever rendered with its default in CI, which is the one value that cannot tell a working template from a broken one. The basic inventory sets 0640 and the job reads the mode back off the installed file. Signed-off-by: Kristofer <kristofer@atlascloud.is> --------- Signed-off-by: Kristofer <kristofer@atlascloud.is>
632 lines
24 KiB
YAML
632 lines
24 KiB
YAML
---
|
|
name: Integration Test
|
|
|
|
on:
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
service_mgr: [systemd, openrc]
|
|
inventory: [basic, notoken]
|
|
exclude:
|
|
- service_mgr: openrc
|
|
inventory: notoken
|
|
env:
|
|
STARTING_K3S_VERSION: v1.34.3+k3s1
|
|
UPGRADE_K3S_VERSION: v1.35.1+k3s1
|
|
SITE_UPGRADE_K3S_VERSION: v1.35.6+k3s1
|
|
# K3s requires privileged containers to run inside Docker and access to cgrougs.
|
|
steps:
|
|
- name: Remove Unnecessary Tools
|
|
working-directory: /
|
|
run: |
|
|
sudo rm -rf \
|
|
/home/linuxbrew \
|
|
/home/packer \
|
|
/opt/az \
|
|
/opt/microsoft \
|
|
/usr/lib/firefox \
|
|
/usr/lib/google-cloud-sdk \
|
|
/usr/local/julia* \
|
|
/usr/local/share/boost \
|
|
/usr/local/share/chromium \
|
|
/usr/local/share/powershell \
|
|
/usr/share/az* \
|
|
/usr/share/dotnet \
|
|
/usr/share/miniconda \
|
|
/usr/share/swift
|
|
df -khl
|
|
- name: Set container OS based on service manager
|
|
id: set-container
|
|
run: |
|
|
if [ "${{ matrix.service_mgr }}" == "systemd" ]; then
|
|
echo "container_os=rancher/systemd-node:v0.0.8" >> $GITHUB_ENV
|
|
echo "container_cmd=/usr/lib/systemd/systemd --unit=noop.target --show-status=true" >> $GITHUB_ENV
|
|
echo "service_mgr=systemd" >> $GITHUB_ENV
|
|
else
|
|
echo "container_os=jrei/openrc-alpine" >> $GITHUB_ENV
|
|
echo "service_mgr=openrc" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible netaddr
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/${{ matrix.inventory }}.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-ansible
|
|
|
|
- name: Start Docker containers
|
|
run: |
|
|
# Start the Server node
|
|
docker run -d --name server-node \
|
|
--hostname server-node \
|
|
--privileged \
|
|
-v /sys/fs/bpf:/sys/fs/bpf \
|
|
--memory 2048m \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ansible \
|
|
${{ env.container_os }} ${{ env.container_cmd }}
|
|
|
|
# Start the Agent node
|
|
docker run -d --name agent-node \
|
|
--hostname agent-node \
|
|
--privileged \
|
|
-v /sys/fs/bpf:/sys/fs/bpf \
|
|
--memory 2048m \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ansible \
|
|
${{ env.container_os }} ${{ env.container_cmd }}
|
|
|
|
- name: Install node dependencies (systemd)
|
|
if: matrix.service_mgr == 'systemd'
|
|
run: |
|
|
docker exec server-node zypper install -y python3-rpm
|
|
docker exec agent-node zypper install -y python3-rpm
|
|
|
|
- name: Install node dependencies (openrc)
|
|
if: matrix.service_mgr == 'openrc'
|
|
run: |
|
|
docker exec server-node apk add curl python3
|
|
docker exec agent-node apk add curl python3
|
|
|
|
- name: Replace k3s_version in inventory
|
|
run: |
|
|
sed -i "s/k3s_version: .*/k3s_version: ${STARTING_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Verify K3s is running on Server
|
|
run: docker exec server-node k3s kubectl get nodes | grep Ready
|
|
|
|
- name: Verify the server config file mode
|
|
if: matrix.inventory == 'basic'
|
|
run: docker exec server-node ls -l /etc/rancher/k3s/config.yaml | grep -q '^-rw-r-----'
|
|
|
|
- name: Verify K3s is running on Agent (systemd)
|
|
if: matrix.service_mgr == 'systemd'
|
|
run: docker exec agent-node systemctl status k3s-agent | grep running
|
|
|
|
- name: Verify K3s is running on Agent (openrc)
|
|
if: matrix.service_mgr == 'openrc'
|
|
run: docker exec agent-node rc-service k3s-agent status | grep started
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Modify the k3s_version in inventory for upgrade
|
|
run: |
|
|
sed -i "s/k3s_version: .*/k3s_version: ${UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Run Upgrade Playbook
|
|
run: ansible-playbook playbooks/upgrade.yml -i tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Verify K3s upgraded on Server
|
|
run: docker exec server-node k3s --version | grep ${UPGRADE_K3S_VERSION}
|
|
|
|
- name: Verify K3s upgraded on Agent
|
|
run: docker exec agent-node k3s --version | grep ${UPGRADE_K3S_VERSION}
|
|
|
|
# Second upgrade, this time re-running site.yml with a bumped version: it
|
|
# must upgrade the running runtime on both the server and the agent, not
|
|
# just replace the on-disk binary.
|
|
- name: Modify the k3s_version in inventory for a site.yml upgrade
|
|
run: |
|
|
sed -i "s/k3s_version: .*/k3s_version: ${SITE_UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Run site Playbook to upgrade via a version bump
|
|
run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml
|
|
|
|
# The install script replaces the on-disk binary regardless of a restart,
|
|
# so assert the running version the kubelet reports to the API: it only
|
|
# advances once the service was actually restarted onto the new build.
|
|
# This is what fails if the version-bump restart regresses.
|
|
- name: Verify the running K3s version upgraded on Server and Agent
|
|
run: |
|
|
for node in server-node agent-node; do
|
|
for attempt in 1 2 3 4 5 6; do
|
|
running=$(docker exec server-node k3s kubectl get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
|
|
echo "$node running version: $running"
|
|
if echo "$running" | grep -q "${SITE_UPGRADE_K3S_VERSION}"; then
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 6 ]; then
|
|
echo "$node did not reach ${SITE_UPGRADE_K3S_VERSION}"
|
|
exit 1
|
|
fi
|
|
sleep 10
|
|
done
|
|
done
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in {1..10}; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 10 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Add node-label to server config
|
|
run: |
|
|
printf " server_config_yaml: |\n node-label: foo=bar\n" >> tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Run site Playbook again to apply new server args
|
|
run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in {1..10}; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 10 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Verify new server args applied
|
|
run: |
|
|
for attempt in 1 2 3; do
|
|
echo "Attempt $attempt: checking node label"
|
|
output=$(docker exec server-node k3s kubectl get nodes -o jsonpath='{.items[*].metadata.labels.foo}' 2>&1)
|
|
echo "$output"
|
|
if echo "$output" | grep -q bar; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 3 ]; then
|
|
sleep 10
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Debug nodes/pods on failure
|
|
if: failure()
|
|
run: |
|
|
echo "NODE INFO"
|
|
docker exec server-node k3s kubectl get nodes -o wide
|
|
echo "POD INFO"
|
|
docker exec server-node k3s kubectl get pods -A -o wide
|
|
echo "CONFIG FILE"
|
|
docker exec server-node cat /etc/rancher/k3s/config.yaml
|
|
echo "Server ENV"
|
|
if [ $service_mgr == "systemd" ]; then
|
|
docker exec server-node cat /etc/systemd/system/k3s.service.env
|
|
elif [ $service_mgr == "openrc" ]; then
|
|
docker exec server-node cat /etc/rancher/k3s/k3s.env
|
|
fi
|
|
echo "Agent ENV"
|
|
if [ $service_mgr == "systemd" ]; then
|
|
docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
|
|
elif [ $service_mgr == "openrc" ]; then
|
|
docker exec agent-node cat /etc/rancher/k3s/k3s-agent.env
|
|
fi
|
|
echo "SERVER LOGS"
|
|
if [ $service_mgr == "systemd" ]; then
|
|
docker exec server-node journalctl -u k3s -n 50
|
|
elif [ $service_mgr == "openrc" ]; then
|
|
docker exec server-node tail -n 50 /var/log/k3s.log
|
|
fi
|
|
echo "AGENT LOGS"
|
|
if [ $service_mgr == "systemd" ]; then
|
|
docker exec agent-node journalctl -u k3s -n 50
|
|
elif [ $service_mgr == "openrc" ]; then
|
|
docker exec agent-node tail -n 50 /var/log/k3s.log
|
|
fi
|
|
|
|
|
|
- name: Remove K3s from Server and Agent
|
|
run: ansible-playbook playbooks/reset.yml -i tests/${{ matrix.inventory }}.yml
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
docker stop server-node && docker rm -f server-node
|
|
docker stop agent-node && docker rm -f agent-node
|
|
docker network rm k3s-ansible
|
|
|
|
test-ha:
|
|
runs-on: ubuntu-latest
|
|
# K3s requires privileged containers to run inside Docker and access to cgrougs.
|
|
env:
|
|
SERVERS: server-node1 server-node2 server-node3
|
|
steps:
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible netaddr
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/ha.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-ha-ansible
|
|
|
|
- name: Start Docker containers
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker run -d --name $SERVER \
|
|
--hostname $SERVER \
|
|
--privileged \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ha-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
done
|
|
|
|
- name: Install server dependencies
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER zypper install -y python3-rpm
|
|
done
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/ha.yml
|
|
|
|
- name: Verify K3s is running on servers
|
|
run: docker exec server-node1 k3s kubectl get nodes | grep Ready | wc -l | grep 3
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Modify the k3s_version in inventory for upgrade
|
|
run: |
|
|
sed -i 's/k3s_version: v1.33.4+k3s1/k3s_version: v1.34.1+k3s1/' tests/ha.yml
|
|
|
|
- name: Run Upgrade Playbook
|
|
run: ansible-playbook playbooks/upgrade.yml -i tests/ha.yml
|
|
|
|
- name: Verify K3s upgraded on all servers
|
|
run: |
|
|
sleep 5 && docker exec server-node1 k3s kubectl get nodes
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER k3s --version | grep v1.34.
|
|
done
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
# Re-running site.yml with a bumped version must upgrade every etcd
|
|
# server. --forks=1 makes Ansible restart them one at a time so the etcd
|
|
# quorum is never lost.
|
|
- name: Modify the k3s_version in inventory for a site.yml upgrade
|
|
run: |
|
|
sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml
|
|
|
|
- name: Run site Playbook to upgrade the servers via a version bump
|
|
run: ansible-playbook playbooks/site.yml -i tests/ha.yml --forks=1
|
|
|
|
# Assert the running (kubelet-reported) version, not the on-disk binary:
|
|
# without a rolling restart the joined servers stay on the old runtime.
|
|
- name: Verify the running K3s version upgraded on all servers via site.yml
|
|
run: |
|
|
docker exec server-node1 k3s kubectl get nodes -o wide
|
|
for SERVER in $SERVERS; do
|
|
for attempt in 1 2 3 4 5 6; do
|
|
running=$(docker exec server-node1 k3s kubectl get node "$SERVER" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
|
|
echo "$SERVER running version: $running"
|
|
if echo "$running" | grep -q v1.34.3; then
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 6 ]; then
|
|
echo "$SERVER did not reach v1.34.3"
|
|
exit 1
|
|
fi
|
|
sleep 10
|
|
done
|
|
done
|
|
|
|
- name: Wait for all deployments to be ready after the site.yml upgrade
|
|
run: |
|
|
for attempt in {1..10}; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 10 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Debug nodes/pods on failure
|
|
if: failure()
|
|
run: |
|
|
echo "NODE INFO"
|
|
docker exec server-node1 k3s kubectl get nodes -o wide
|
|
echo "POD INFO"
|
|
docker exec server-node1 k3s kubectl get pods -A -o wide
|
|
echo "CONFIG FILES"
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER cat /etc/rancher/k3s/config.yaml
|
|
done
|
|
echo "Server ENVS"
|
|
for SERVER in $SERVERS; do
|
|
docker exec $SERVER cat /etc/systemd/system/k3s.service.env
|
|
done
|
|
echo "SERVER LOGSS"
|
|
docker exec server-node1 journalctl -u k3s -n 10
|
|
|
|
- name: Remove K3s from server nodes
|
|
run: ansible-playbook playbooks/reset.yml -i tests/ha.yml
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
for SERVER in $SERVERS; do
|
|
docker stop $SERVER && docker rm -f $SERVER
|
|
done
|
|
docker network rm k3s-ha-ansible
|
|
|
|
test-external:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible netaddr
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/external.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-ext-ansible
|
|
|
|
- name: Start Database container
|
|
run: |
|
|
docker run -d --name mysqlDB \
|
|
--network=k3s-ext-ansible \
|
|
-p 3306:3306 -e MYSQL_ROOT_PASSWORD=e2e mysql:8.4
|
|
|
|
- name: Start Server container
|
|
run: |
|
|
docker run -d --name server-node \
|
|
--privileged \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-ext-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
|
|
- name: Install server dependencies
|
|
run: docker exec server-node zypper install -y python3-rpm
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/external.yml
|
|
|
|
- name: Verify K3s is running on servers
|
|
run: docker exec server-node k3s kubectl get nodes | grep Ready | wc -l | grep 1
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
docker stop server-node && docker rm -f server-node
|
|
docker stop mysqlDB && docker rm -f mysqlDB
|
|
docker network rm k3s-ext-ansible
|
|
|
|
test-airgap:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout codebase
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
|
|
|
|
- name: Set up Python 3.13.
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version: '3.13.x'
|
|
|
|
- name: Install Ansible and dependencies
|
|
run: |
|
|
pip install ansible netaddr
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Verify Inventory
|
|
run: ansible-inventory -i tests/airgap.yml --list
|
|
|
|
- name: Create Docker Network
|
|
run: docker network create k3s-airgp-ansible
|
|
|
|
- name: Start containers
|
|
run: |
|
|
docker run -d --name server-node \
|
|
--privileged \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-airgp-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
|
|
docker run -d --name agent-node \
|
|
--privileged \
|
|
--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--volume=/lib/modules:/lib/modules:ro \
|
|
--cgroupns=host \
|
|
--network=k3s-airgp-ansible \
|
|
rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
|
|
|
|
- name: Install node dependencies
|
|
run: |
|
|
docker exec server-node zypper install -y python3-rpm
|
|
docker exec agent-node zypper install -y python3-rpm
|
|
|
|
- name: Download K3s airgap artifacts
|
|
run: |
|
|
mkdir -p ./playbooks/test-airgap
|
|
wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s-airgap-images-amd64.tar.gz
|
|
wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s
|
|
|
|
- name: Run Playbook
|
|
env:
|
|
ANSIBLE_FORCE_COLOR: '1'
|
|
run: ansible-playbook playbooks/site.yml -i tests/airgap.yml
|
|
|
|
- name: Verify K3s is running on servers
|
|
run: docker exec server-node k3s kubectl get nodes | grep Ready | wc -l | grep 2
|
|
|
|
- name: Validate its the correct K3s version
|
|
run: |
|
|
docker exec server-node k3s --version | grep v1.33.8
|
|
docker exec agent-node k3s --version | grep v1.33.8
|
|
|
|
- name: Wait for all deployments to be ready
|
|
run: |
|
|
for attempt in 1 2 3 4 5 6; do
|
|
echo "Attempt $attempt: checking deployments"
|
|
output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
|
|
echo "$output"
|
|
if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -lt 6 ]; then
|
|
sleep 15
|
|
fi
|
|
done
|
|
exit 1
|
|
|
|
- name: Debug nodes/pods on failure
|
|
if: failure()
|
|
run: |
|
|
echo "NODE INFO"
|
|
docker exec server-node k3s kubectl get nodes -o wide
|
|
echo "POD INFO"
|
|
docker exec server-node k3s kubectl get pods -A -o wide
|
|
echo "CONFIG FILE"
|
|
docker exec server-node cat /etc/rancher/k3s/config.yaml
|
|
echo "Server ENV"
|
|
docker exec server-node cat /etc/systemd/system/k3s.service.env
|
|
echo "Agent ENV"
|
|
docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
|
|
echo "SERVER LOGS"
|
|
docker exec server-node journalctl -u k3s -n 10
|
|
|
|
- name: Stop and remove Docker containers
|
|
run: |
|
|
docker stop server-node && docker rm -f server-node
|
|
docker stop agent-node && docker rm -f agent-node
|
|
docker network rm k3s-airgp-ansible
|
|
|