From fe4407db0ea92763128ac02eb2dd2f31a5dfa1e1 Mon Sep 17 00:00:00 2001 From: kip Date: Wed, 18 Oct 2023 19:01:10 +0200 Subject: [PATCH 1/3] Added verify_config.yml playbook and included it in site.yml, Updated README.md with config verifications for ip routing prefix and metal_lb_ip_range checks etc --- .gitignore | 1 + README.md | 15 +++ inventory/sample/group_vars/all.yml | 3 +- roles/verify_config/tasks/main.yml | 13 ++ .../__pycache__/range_to_ips.cpython-310.pyc | Bin 0 -> 1694 bytes .../filter_plugins/range_to_ips.py | 53 ++++++++ roles/verify_config_gather/tasks/main.yml | 117 ++++++++++++++++++ site.yml | 3 + verify_config.yml | 14 +++ 9 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 roles/verify_config/tasks/main.yml create mode 100644 roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc create mode 100644 roles/verify_config_gather/filter_plugins/range_to_ips.py create mode 100644 roles/verify_config_gather/tasks/main.yml create mode 100644 verify_config.yml diff --git a/.gitignore b/.gitignore index 78f3d0b..4ed5497 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env/ *.log ansible.cfg +__pycache__ diff --git a/README.md b/README.md index cdb24fd..8c09e18 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,21 @@ This requires at least k3s version `1.19.1` however the version is configurable If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment. +#### verify routing prefixes and ip addresses in all.yml +Ensure that the ip routing prefix matches the the one from the flannel_iface (`eth0` by default) for each host. +Example `192.168.30` is the default routing prefix, and it is used in the following variables: +- apiserver_endpoint +- metal_lb_bgp_peer_address (commented by default) +- metal_lb_ip_range +Also Ensure that the apiserver_endpoint is not in the metal_lb_ip_range + +For your convience The playbook site.yml verifies the above elements of the config are valid. +* Optionally to skip these verifications set, in all.yml `verify_config: false` +* Optionally to just verify the config Run + `ansible-playbook -i inventory/my-cluster/hosts.ini verify_config.yml` +* Optionally manually recover ip routing prefix based on inet and netmask from + `ansible -i inventory/my-cluster/hosts.ini --become -m shell -a 'ifconfig eth0 | grep "inet "' all` + ### ☸️ Create Cluster Start provisioning of the cluster using the following command: diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 4b1f2da..12d22eb 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -82,7 +82,6 @@ proxmox_lxc_ct_ids: - 203 - 204 -# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache # (harbor / nexus / docker's official registry / etc). # Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub), # or air-gapped environments where your nodes don't have internet access after the initial setup @@ -121,3 +120,5 @@ custom_registries_yaml: | auth: username: yourusername password: yourpassword + +verify_config: true \ No newline at end of file diff --git a/roles/verify_config/tasks/main.yml b/roles/verify_config/tasks/main.yml new file mode 100644 index 0000000..facdfec --- /dev/null +++ b/roles/verify_config/tasks/main.yml @@ -0,0 +1,13 @@ +--- +# To be ran on localhost after verify_config_gather role. +# Verifies gatherd facts. +- name: Collect all routing4_prefixes into a list + set_fact: + all_routing4_prefixes: "{{ groups['all'] | map('extract', hostvars, 'routing4_prefix') | list }}" + routing4_prefix: "{{ groups['all'] | map('extract', hostvars, 'routing4_prefix') | list | first }}" + +- name: Ensure all hosts have the same routing4_prefix + assert: + that: all_routing4_prefixes | unique | length == 1 + fail_msg: "Not all hosts have the same routing4_prefix." + success_msg: "Using verified routing prefix {{ routing4_prefix }} across all hosts" diff --git a/roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc b/roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c874cf3a722c34747e0db397fc5494e2babf61a GIT binary patch literal 1694 zcmZ`3%WfMtkmRnU)x$;`IFGg{DikOR71eH}07W7gMv4~c1E?rrO3uc1W--IzayXoKzO&OI0MGvm4olT-e+P z)V~4|M9_pBkv0)bIM2ut6|QK&%0yGNV0C^W!V~RhM6{*5c8(gdDLThYeJ$CVP`G_) zanOAO+Q>9=aM=Lzh(7`98$b+MkriFD6+33S{X2O|1vw-GI(<#cTbVSW5Q@hIk25Yp zz2GX$=5pYaEH281-ISjEr;w46;x$p=9eMj#u*g#xWJMeZc^Vww(TKw^)A7SZ4o#c} zDo>;iPNj-xXFSTYSv=?S&@7}1W^rPq;ze>Yk24+EH@L~c2<;bVrE8=%_>5*iZcx>R zB?9~aFrwDS0vH4Fz*w#wTJf~TLu}9gH_!PNM0hLT60I8FuE})H`L5_yyp7(@blDx< z8~dXl?)oEtw7gl{4BzwPKkFys+R2>wEX7szu~1{cdrl|UnO?= zh4Hd6nT#i+W!ElnX*?MZs(9Nx*N6vEF2BUg@Q>9Dc7Wak(qlbpKWDqTRN-la3EcuF zunB0BXxoCW=qbFuqE8s0lkKkPOGxZ1NbP3?|7~^-?3@*;n#RzVP}(c!75ka2=_!_W zbNZGJ6$RaIkVHtffEM7(hk(}DIkd;$%_kJvgs$le_Lv=3MLwXVqm3G{lKHn4-nkf# z&VCYW6Xj`f@GYuC-FsyD{l^02?hpfp$lCA!mVwE$Fp_O5I+nT|W6p*(V#^HcXt&0jzA#<*l z$?QUO%GSo-0Aw^K)Mf6w{l erMF*Mc7UPL*MNA0G0*iJ@A@ZQuhXIKyZ-_C`*Do` literal 0 HcmV?d00001 diff --git a/roles/verify_config_gather/filter_plugins/range_to_ips.py b/roles/verify_config_gather/filter_plugins/range_to_ips.py new file mode 100644 index 0000000..0787fc5 --- /dev/null +++ b/roles/verify_config_gather/filter_plugins/range_to_ips.py @@ -0,0 +1,53 @@ +import netaddr + +''' returns True if ip is in range + examples: + - see test functions below +''' +def test(ip, range, expected): + assert netaddr_ip_in_dash_range(ip, range) == expected + +def test_netaddr_ip_in_dash_range(): + # ipv4 + test('192.168.1.1', '192.168.1.1-192.168.1.2', True) + test('192.168.1.1', '192.168.1.1', True) + test('192.168.1.1', '192.168.1.2-192.168.1.3', False) + test('192.168.1.1', '192.168.1.2', False) + + # ipv6 style + test('::ffff:192.168.1.1', '::ffff:192.168.1.1-::ffff:192.168.1.8', True) + test('::ffff:192.168.1.1', '::ffff:192.168.1.1', True) + test('::ffff:192.168.1.1', '::ffff:192.168.1.2-::ffff:192.168.1.8', False) + test('::ffff:192.168.1.1', '::ffff:192.168.1.2', False) + + # Note I expedted true but apperently the netaddr library does not support this?? or I don't understand ipv6 :) + test('::2:1', '::2:1-::2:2', False) + ''' + todo: ? + - netaddr_ip_in_dash_range('192.168.1.1', '192.168.1.0/24') => True (TODO: test, implement) + - netaddr_ip_in_dash_range('192.168.99.1', '192.168.1.0/24') => False (TODO: test, implement) + ''' + +def netaddr_ip_in_dash_range(ip, range): + # return False early if range is invalid + if '-' not in range: + ip_start = range + ip_end = range + else: + ip_start = range.split('-')[0] + ip_end = range.split('-')[1] + return ip in [str(ip) for ip in netaddr.iter_iprange(ip_start, ip_end)] + + +class FilterModule(object): + ''' Ansible filters. Interface to custom netaddr methods. + https://pypi.org/project/netaddr/ + ''' + + def filters(self): + return { + 'netaddr_ip_in_dash_range' : netaddr_ip_in_dash_range + } + +if __name__ == '__main__': + test_netaddr_ip_in_dash_range() diff --git a/roles/verify_config_gather/tasks/main.yml b/roles/verify_config_gather/tasks/main.yml new file mode 100644 index 0000000..5b4cb72 --- /dev/null +++ b/roles/verify_config_gather/tasks/main.yml @@ -0,0 +1,117 @@ +--- +# To be ran on all hosts before role verify_config. +# Gathers facts and ensures they are set + +- name: Set routing4_prefix from regex + set_fact: + routing4_prefix: "{{ hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv4']['broadcast'] + | regex_replace('\\.?255', '') }}" + routing6_cidr: "{{ hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv6'][0]['address'] }}/{{ + hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv6'][0]['prefix'] }}" + +- name: Check if fact routing4_prefix exists and is not empty + assert: + that: + - routing4_prefix is defined + - routing4_prefix is not none + - routing4_prefix != '' + fail_msg: >- + The fact 'routing_4prefix' is not defined, is null, or is empty + (based on flannel_iface: {{ flannel_iface }} ipv4 broadcast). + +# metal_lb_bgp_peer_address +- name: Assert that metal_lb_bgp_peer_address starts with routing4_prefix, or is ipv6 + assert: + that: + - > + metal_lb_bgp_peer_address.startswith(routing4_prefix) + or metal_lb_bgp_peer_address.matches('.*:.*') + fail_msg: > + The fact 'metal_lb_bgp_peer_address' <{{ metal_lb_bgp_peer_address }}> + doesn't start with the routing prefix <{{ routing4_prefix }}> + when: metal_lb_bgp_peer_address is defined + +# metal_lb_ip_range +- name: > + Assert that metal_lb_ip_range (when string) contains - + and both ips start with routing4_prefix, skip any containing ':' (ipv6) + assert: + that: + - metal_lb_ip_range | regex_search('^{{ routing4_prefix }}\.[0-9]{1,3}-{{ routing4_prefix }}\.[0-9]{1,3}$|.*:.*') + fail_msg: > + metal_lb_ip_range <{{ metal_lb_ip_range }}> has one or more ipv4s + that don't start with the routing prefix <{{ routing4_prefix }}> + when: metal_lb_ip_range is string + +- name: Assert that metal_lb_ip_ranges (when list) has only strings that match the regexes in the task above + assert: + that: + - > + ( metal_lb_ip_range + | select('match', '^{{ routing4_prefix }}\.[0-9]{1,3}-{{ routing4_prefix }}\.[0-9]{1,3}$|.*:.*') + | list | length + ) + == + (metal_lb_ip_range + | length + ) + fail_msg: > + metal_lb_ip_range <{{ metal_lb_ip_range }}> has one or more values with ipv4s + that don't start with the routing prefix <{{ routing4_prefix }}> + when: metal_lb_ip_range is not string and metal_lb_ip_range is not mapping and metal_lb_ip_range is iterable + +# apiserver_endpoint +- name: Assert that apiserver_endpoint is not in metal_lb_ip_range (when string) using network_in_usable + # For / ranges + assert: + that: + - not ( metal_lb_ip_range | ansible.utils.network_in_usable( apiserver_endpoint )) + fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range {{ metal_lb_ip_range }}" + success_msg: > + apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range {{ metal_lb_ip_range }} + when: metal_lb_ip_range is string + +- name: > + Assert that apiserver_endpoint is not in metal_lb_ip_range (when string) + using custom filter netaddr_ip_in_dash_range + # For - ranges. Not sure this works for ipv6 + assert: + that: + - not (apiserver_endpoint | netaddr_ip_in_dash_range(metal_lb_ip_range)) + fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range {{ metal_lb_ip_range }}" + success_msg: > + apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range {{ metal_lb_ip_range }} + when: metal_lb_ip_range is string + # *probably* in the success_msg sections of the prior two tasks because not all cases may work + +- name: Assert that apiserver_endpoint is not in metal_lb_ip_ranges (when list) using logic of the task above + assert: + that: + - not (apiserver_endpoint | netaddr_ip_in_dash_range(item)) + # this probably fails on an ipv6 range ''::1-::2', and on / ranges + fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range item {{ item }}" + success_msg: > + apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range item {{ item }} + loop: "{{ metal_lb_ip_range | list }}" + when: metal_lb_ip_range is not string and metal_lb_ip_range is not mapping and metal_lb_ip_range is iterable + # these (when string, when list tasks) smell funny. + # It seems there should be a way in one task to handle an object that is a string or a list + # and loop on {{ [metal_lb_ip_range] }} or {{ metal_lb_ip_range }} respectively + # when it is a string it skips each char :( + # I tried the select pattern like in the task + # 'assert that metal_lb_ip_ranges (when list) has only strings that match the regexes in the task above' + # but it didn't work + +- name: Assert that apiserver_endpoint, is ipv4 and starts with routing4_prefix, or is ipv6 and is in routing6_cidr + assert: + that: + - apiserver_endpoint is defined + - apiserver_endpoint is not none + - >- + ( apiserver_endpoint | ansible.utils.ipv4 and apiserver_endpoint.startswith(routing4_prefix) ) + or + ( apiserver_endpoint | ansible.utils.ipv6 and apiserver_endpoint | ansible.utils.ipaddr(routing6_cidr)) + fail_msg: > + The fact 'apiserver_endpoint' <{{ apiserver_endpoint }}> + doesn't start with the routing prefix <{{ routing4_prefix }}> + or is not in the routing6_cidr <{{ routing6_cidr }}> diff --git a/site.yml b/site.yml index 33653a9..c6d0d5f 100644 --- a/site.yml +++ b/site.yml @@ -1,4 +1,7 @@ --- +- name: Import verify_config playbook + ansible.builtin.import_playbook: verify_config.yml + - name: Prepare Proxmox cluster hosts: proxmox gather_facts: true diff --git a/verify_config.yml b/verify_config.yml new file mode 100644 index 0000000..0ca2581 --- /dev/null +++ b/verify_config.yml @@ -0,0 +1,14 @@ +--- +- name: Gather config for verify + hosts: all + gather_facts: true + roles: + - role: verify_config_gather + when: verify_config is not defined or verify_config + +- name: Verify config + hosts: localhost + gather_facts: false + roles: + - role: verify_config + when: verify_config is not defined or verify_config From 13deec85b347ef342b887e9af349aa67c554feb9 Mon Sep 17 00:00:00 2001 From: kip Date: Thu, 19 Oct 2023 18:51:33 +0200 Subject: [PATCH 2/3] fixed newline --- inventory/sample/group_vars/all.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 12d22eb..e6cec5f 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -82,6 +82,7 @@ proxmox_lxc_ct_ids: - 203 - 204 +# Only enable this if you have set up your own container registry to act as a mirror / pull-through cache # (harbor / nexus / docker's official registry / etc). # Can be beneficial for larger dev/test environments (for example if you're getting rate limited by docker hub), # or air-gapped environments where your nodes don't have internet access after the initial setup @@ -121,4 +122,4 @@ custom_registries_yaml: | username: yourusername password: yourpassword -verify_config: true \ No newline at end of file +verify_config: true From e3a95bb5a2dd1c23a1879e5b39dc00aaa1a52184 Mon Sep 17 00:00:00 2001 From: kip Date: Sun, 22 Oct 2023 12:58:21 +0200 Subject: [PATCH 3/3] removed file that should have been ignored, and fixed .gitignore --- .gitignore | 4 +++- .../__pycache__/range_to_ips.cpython-310.pyc | Bin 1694 -> 0 bytes 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc diff --git a/.gitignore b/.gitignore index 4ed5497..fa25d83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env/ *.log ansible.cfg -__pycache__ + +__pycache__/ +*.py[cod] diff --git a/roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc b/roles/verify_config_gather/filter_plugins/__pycache__/range_to_ips.cpython-310.pyc deleted file mode 100644 index 3c874cf3a722c34747e0db397fc5494e2babf61a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1694 zcmZ`3%WfMtkmRnU)x$;`IFGg{DikOR71eH}07W7gMv4~c1E?rrO3uc1W--IzayXoKzO&OI0MGvm4olT-e+P z)V~4|M9_pBkv0)bIM2ut6|QK&%0yGNV0C^W!V~RhM6{*5c8(gdDLThYeJ$CVP`G_) zanOAO+Q>9=aM=Lzh(7`98$b+MkriFD6+33S{X2O|1vw-GI(<#cTbVSW5Q@hIk25Yp zz2GX$=5pYaEH281-ISjEr;w46;x$p=9eMj#u*g#xWJMeZc^Vww(TKw^)A7SZ4o#c} zDo>;iPNj-xXFSTYSv=?S&@7}1W^rPq;ze>Yk24+EH@L~c2<;bVrE8=%_>5*iZcx>R zB?9~aFrwDS0vH4Fz*w#wTJf~TLu}9gH_!PNM0hLT60I8FuE})H`L5_yyp7(@blDx< z8~dXl?)oEtw7gl{4BzwPKkFys+R2>wEX7szu~1{cdrl|UnO?= zh4Hd6nT#i+W!ElnX*?MZs(9Nx*N6vEF2BUg@Q>9Dc7Wak(qlbpKWDqTRN-la3EcuF zunB0BXxoCW=qbFuqE8s0lkKkPOGxZ1NbP3?|7~^-?3@*;n#RzVP}(c!75ka2=_!_W zbNZGJ6$RaIkVHtffEM7(hk(}DIkd;$%_kJvgs$le_Lv=3MLwXVqm3G{lKHn4-nkf# z&VCYW6Xj`f@GYuC-FsyD{l^02?hpfp$lCA!mVwE$Fp_O5I+nT|W6p*(V#^HcXt&0jzA#<*l z$?QUO%GSo-0Aw^K)Mf6w{l erMF*Mc7UPL*MNA0G0*iJ@A@ZQuhXIKyZ-_C`*Do`