From 565c9fa0495a5d82893c87c113c416785b47b205 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 8 Nov 2023 14:00:52 -0800 Subject: [PATCH] Enforce use of a defined token. Simplifies additional server and agent joining process. Signed-off-by: Derek Nola --- Vagrantfile | 16 +++++----- inventory-sample.yml | 4 +-- roles/k3s/agent/defaults/main.yml | 1 + .../k3s/agent/templates/k3s-agent.service.j2 | 2 +- roles/k3s/server/defaults/main.yml | 1 + roles/k3s/server/tasks/main.yml | 32 ++----------------- .../server/templates/k3s-server.service.j2 | 2 +- 7 files changed, 16 insertions(+), 42 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 795daa8..80cee41 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,13 +9,16 @@ NETWORK_PREFIX = "10.10.10" def provision(vm, role, node_num) vm.box = NODE_BOXES[node_num] vm.hostname = role - # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 + # We use a private network because the default IPs are dynamicly assigned + # during provisioning. This makes it impossible to know the server-0 IP when + # provisioning subsequent servers and agents. A private network allows us to + # assign static IPs to each node, and thus provide a known IP for the API endpoint. node_ip = "#{NETWORK_PREFIX}.#{100+node_num}" + # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 vm.network "private_network", ip: node_ip, netmask: "255.255.255.0" vm.provision "ansible", run: 'once' do |ansible| ansible.compatibility_mode = "2.0" - ansible.verbose = "vv" ansible.playbook = "playbook/site.yml" ansible.groups = { "server" => NODE_ROLES.grep(/^server/), @@ -25,13 +28,12 @@ def provision(vm, role, node_num) ansible.extra_vars = { k3s_version: "v1.26.5+k3s1", api_endpoint: "#{NETWORK_PREFIX}.100", - api_port: 6443, - extra_server_args: "", - extra_server_init_args: "", + token: "myyagrant", + # Required to use the private network configured above + extra_server_args: "--node-external-ip #{node_ip} --flannel-iface eth1", extra_agent_args: "", } end - end Vagrant.configure("2") do |config| @@ -45,8 +47,6 @@ Vagrant.configure("2") do |config| v.memory = NODE_MEMORY end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves NODE_ROLES.each_with_index do |name, i| config.vm.define name do |node| provision(node.vm, name, i) diff --git a/inventory-sample.yml b/inventory-sample.yml index cf46422..afeca60 100644 --- a/inventory-sample.yml +++ b/inventory-sample.yml @@ -14,12 +14,12 @@ k3s_cluster: ansible_port: 22 ansible_user: debian k3s_version: v1.25.5+k3s2 + token: "mytoken" # Use ansible vault if you want to keep it secret api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" - api_port: 6443 extra_server_args: "" - extra_server_init_args: "" extra_agent_args: "" # Optional vars + # api_port: 6443 # k3s_server_location: /var/lib/rancher/k3s # systemd_dir: /etc/systemd/system diff --git a/roles/k3s/agent/defaults/main.yml b/roles/k3s/agent/defaults/main.yml index e0b678e..cbcb1e9 100644 --- a/roles/k3s/agent/defaults/main.yml +++ b/roles/k3s/agent/defaults/main.yml @@ -1,3 +1,4 @@ --- k3s_server_location: "/var/lib/rancher/k3s" systemd_dir: "/etc/systemd/system" +api_port: 6443 diff --git a/roles/k3s/agent/templates/k3s-agent.service.j2 b/roles/k3s/agent/templates/k3s-agent.service.j2 index a806bbb..e0157b2 100644 --- a/roles/k3s/agent/templates/k3s-agent.service.j2 +++ b/roles/k3s/agent/templates/k3s-agent.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ hostvars[groups['server'][0]]['token'] }} {{ extra_agent_args }} +ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead diff --git a/roles/k3s/server/defaults/main.yml b/roles/k3s/server/defaults/main.yml index e0b678e..cbcb1e9 100644 --- a/roles/k3s/server/defaults/main.yml +++ b/roles/k3s/server/defaults/main.yml @@ -1,3 +1,4 @@ --- k3s_server_location: "/var/lib/rancher/k3s" systemd_dir: "/etc/systemd/system" +api_port: 6443 diff --git a/roles/k3s/server/tasks/main.yml b/roles/k3s/server/tasks/main.yml index 8028378..2d8b0b0 100644 --- a/roles/k3s/server/tasks/main.yml +++ b/roles/k3s/server/tasks/main.yml @@ -6,7 +6,7 @@ ansible.builtin.command: cmd: > systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server - --cluster-init --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args}} + --cluster-init --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args}} # noqa: jinja[spacing] creates: "{{ k3s_server_location }}/server/node-token" when: groups['server'] | length > 1 @@ -15,38 +15,10 @@ ansible.builtin.command: cmd: > systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server - --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }} + --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }} creates: "{{ k3s_server_location }}/server/node-token" when: groups['server'] | length == 1 - - name: Wait for node-token - ansible.builtin.wait_for: - path: "{{ k3s_server_location }}/server/node-token" - - - name: Register node-token file access mode - ansible.builtin.stat: - path: "{{ k3s_server_location }}/server/node-token" - register: p - - - name: Change file access node-token - ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" - mode: "g+rx,o+rx" - - - name: Read node-token from server - ansible.builtin.slurp: - path: "{{ k3s_server_location }}/server/node-token" - register: node_token - - - name: Store server node-token - ansible.builtin.set_fact: - token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" - - - name: Restore node-token file access - ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" - mode: "{{ p.stat.mode }}" - - name: Create directory .kube ansible.builtin.file: path: ~{{ ansible_user }}/.kube diff --git a/roles/k3s/server/templates/k3s-server.service.j2 b/roles/k3s/server/templates/k3s-server.service.j2 index 6e898eb..92a1220 100644 --- a/roles/k3s/server/templates/k3s-server.service.j2 +++ b/roles/k3s/server/templates/k3s-server.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args }} +ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args }} --token {{ token }} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead