Mercurial > code > home > repos > infra
changeset 89:2fddde57231b
no connman to surprisingly rewrite net configs
| author | drewp@bigasterisk.com |
|---|---|
| date | Sun, 10 Jul 2022 19:51:16 -0700 |
| parents | dae714e8f620 |
| children | 376ab3be6e94 |
| files | kube.py net.py templates/dnsmasq/dnsmasq.conf.j2 templates/kube/config.yaml.j2 templates/kube/coredns.yaml templates/kube/node-config.yaml.j2 templates/net/house_net.service.j2 templates/wireguard/wg0.conf.j2 wireguard.py |
| diffstat | 9 files changed, 85 insertions(+), 61 deletions(-) [+] |
line wrap: on
line diff
--- a/kube.py Sun Jul 10 19:50:52 2022 -0700 +++ b/kube.py Sun Jul 10 19:51:16 2022 -0700 @@ -1,3 +1,4 @@ +import os import tempfile from pyinfra import host from pyinfra.facts.files import FindInFile @@ -5,19 +6,19 @@ from pyinfra.operations import files, server, systemd is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux'] - +raise NotImplementedError("update templates from current config files") # https://github.com/k3s-io/k3s/releases -k3s_version = 'v1.24.2-rc1+k3s1' +# 1.23.6 per https://github.com/cilium/cilium/issues/20331 +k3s_version = 'v1.23.6+k3s1' # https://github.com/GoogleContainerTools/skaffold/releases -skaffold_version = 'v1.39.0' +skaffold_version = 'v1.39.1' master_ip = "10.5.0.1" server_node = 'bang' -nodes = ['slash', 'dash', 'frontbed', 'garage'] +nodes = ['slash', 'dash'] #, 'dash', 'frontbed', 'garage'] admin_from = ['bang', 'slash', 'dash'] - -if host.name in nodes + [server_node]: +def host_prep(): server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True) server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True) server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True) @@ -41,21 +42,30 @@ files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline) # pi needs reboot now + # https://github.com/k3s-io/k3s/issues/1812 unclear server.shell(commands=[ - 'update-alternatives --set iptables /usr/sbin/iptables-nft', - 'update-alternatives --set ip6tables /usr/sbin/ip6tables-nft', + 'update-alternatives --set iptables /usr/sbin/iptables-legacy', + 'update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy', ]) # needs reboot if this changed # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/ files.directory(path='/etc/rancher/k3s') - files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml') + +def config_and_run_service(): + service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service' + which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2' + role = 'server' if host.name == server_node else 'agent' - service_name = 'k3s.service' if host.name == 'bang' else 'k3s-node.service' - which_conf = 'config.yaml.j2' if host.name == 'bang' else 'node-config.yaml.j2' - - # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token - token = open('secrets/k3s_token', 'rt').read().strip() + # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token, + # so this presumes a previous run + if host.name == server_node: + token="ununsed" + else: + if not os.path.exists('/var/lib/rancher/k3s/server/node-token'): + print("first pass is for server only- skipping other nodes") + return + token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip() files.template( src=f'templates/kube/{which_conf}', dest='/etc/k3s_config.yaml', @@ -63,29 +73,25 @@ token=token, wg_ip=host.host_data['wireguard_address'], ) - files.template( - src='templates/kube/k3s_resolv.conf.j2', - dest='/etc/k3s_resolv.conf', - master_ip=master_ip, - wg_ip=host.host_data['wireguard_address'], - ) - files.template( - src='templates/kube/k3s_flannel.conf.j2', - dest='/etc/k3s_flannel.conf', - master_ip=master_ip, - wg_ip=host.host_data['wireguard_address'], - ) - files.put( - src='templates/kube/flannel.link', # - dest='/etc/systemd/network/10-flannel.link') # then reboot + # files.put( + # src='templates/kube/flannel.link', # + # dest='/etc/systemd/network/10-flannel.link') # then reboot files.template( src='templates/kube/k3s.service.j2', dest=f'/etc/systemd/system/{service_name}', - role='server' if host.name == 'bang' else 'agent', + role=role, ) systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True) -if host.name == 'bang': + +if host.name in nodes + [server_node]: + host_prep() + + # not until registry is up, right? + files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml') + config_and_run_service() + +if host.name == server_node: files.put( src="templates/kube/coredns.yaml", dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml", @@ -121,8 +127,8 @@ files.directory(path='/home/drewp/.kube', user='drewp', group='drewp') files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml') - files.chown(target='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp') - files.chmod(target='/etc/rancher/k3s/k3s.yaml', mode='640') + # assumes pyinfra is running on server_node + files.put(src='/etc/rancher/k3s/k3s.yaml', dest='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640') # see https://github.com/GoogleContainerTools/skaffold/releases files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64', @@ -131,3 +137,5 @@ group='root', mode='755', cache_time=1000) + # one time; writes to $HOME + #skaffold config set --global insecure-registries bang5:5000 \ No newline at end of file
--- a/net.py Sun Jul 10 19:50:52 2022 -0700 +++ b/net.py Sun Jul 10 19:51:16 2022 -0700 @@ -22,7 +22,7 @@ delete=True, ) - apt.packages(packages=['network-manager'], present=False) + apt.packages(packages=['network-manager', 'connman'], present=False) # On bang: # Now using a HW router for this firewall. No incoming connections.
--- a/templates/dnsmasq/dnsmasq.conf.j2 Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/dnsmasq/dnsmasq.conf.j2 Sun Jul 10 19:51:16 2022 -0700 @@ -4,6 +4,9 @@ listen-address={{ net }}.0.1 {% if net == "10.2" %} +# dnsmasq will not automatically listen on the loopback interface. To achieve +# this, its IP address, 127.0.0.1, must be explicitly given as a +# --listen-address option. listen-address=127.0.0.1 {% endif %} bind-interfaces
--- a/templates/kube/config.yaml.j2 Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/kube/config.yaml.j2 Sun Jul 10 19:51:16 2022 -0700 @@ -1,11 +1,18 @@ -debug: false +#debug: true write-kubeconfig-mode: '640' -bind-address: 0.0.0.0 -http-listen-port: 6443 + +#bind-address: 0.0.0.0 +#http-listen-port: 6443 #{{ master_ip }} -node-ip: {{ wg_ip }} -flannel-backend: host-gw -flannel-iface: ens5 +#node-ip: {{ wg_ip }} +#flannel-backend: host-gw +#flannel-iface: wg0 #flannel-conf: /etc/k3s_flannel.conf disable: - - traefik \ No newline at end of file + - traefik + +node-ip: {{ wg_ip }} +flannel-backend: none +disable-network-policy: true +cluster-cidr: 172.16.0.0/16 +service-cidr: 172.26.0.0/16 \ No newline at end of file
--- a/templates/kube/coredns.yaml Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/kube/coredns.yaml Sun Jul 10 19:51:16 2022 -0700 @@ -212,7 +212,7 @@ spec: selector: k8s-app: kube-dns - clusterIP: '10.43.0.10' + clusterIP: '172.26.0.10' ports: - name: dns port: 53
--- a/templates/kube/node-config.yaml.j2 Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/kube/node-config.yaml.j2 Sun Jul 10 19:51:16 2022 -0700 @@ -1,6 +1,8 @@ -debug: false -node-external-ip: {{ wg_ip }} +#debug: true +#node-external-ip: {{ wg_ip }} node-ip: {{ wg_ip }} token: {{ token }} server: https://{{ master_ip }}:6443 -resolv-conf: /etc/k3s_resolv.conf +#resolv-conf: /etc/k3s_resolv.conf +# flannel-iface: wg0 +no-flannel: true \ No newline at end of file
--- a/templates/net/house_net.service.j2 Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/net/house_net.service.j2 Sun Jul 10 19:51:16 2022 -0700 @@ -4,7 +4,8 @@ [Service] Type=oneshot -ExecStart=/usr/sbin/iptables -A POSTROUTING --table nat --out-interface {{out_interface}} --jump MASQUERADE +# haven't yet debugged why this doesn't work right away +ExecStart=bash -c "sleep 10; /usr/sbin/iptables -A POSTROUTING --table nat --out-interface {{out_interface}} --jump MASQUERADE" [Install] WantedBy=multi-user.target
--- a/templates/wireguard/wg0.conf.j2 Sun Jul 10 19:50:52 2022 -0700 +++ b/templates/wireguard/wg0.conf.j2 Sun Jul 10 19:51:16 2022 -0700 @@ -6,22 +6,25 @@ PrivateKey = {{priv_key}} ListenPort = 1195 +# suggested by https://i.reddit.com/r/WireGuard/comments/jcwleo/ubuntu_2004_lts_server_as_wireguard_client/ +#FwMark = 0x4000 + {% if host.name == 'bang' %} - {{ peer_block('dash', 'X39ewB2uYLZTFaG+RFeLpyOrnCgjc4wRKrcV0Jz3sTM=', '10.5.0.5/32', 'dash:1195') }} - {{ peer_block('dot', 'sav1VQE1XzbOGfNjDRxcHAmEWtmVGYC1B7KXH+5IKxY=', '10.5.0.30/32', 'dot:1195') }} - {{ peer_block('frontbed', 'ENhRhEgGaFfwV74MqYBHJgkOFpNAF5kVHVK5/tRVTjU=', '10.5.0.17/32', 'frontbed:1195') }} - {{ peer_block('garage', 'kFMtVafPU8kJHYmdafc1g/OLRnNPQMGpYKcDqQ9rUjA=', '10.5.0.14/32', 'garage:1195') }} + {{ peer_block('dash', 'X39ewB2uYLZTFaG+RFeLpyOrnCgjc4wRKrcV0Jz3sTM=', '10.5.0.5/32') }} + {{ peer_block('dot', 'sav1VQE1XzbOGfNjDRxcHAmEWtmVGYC1B7KXH+5IKxY=', '10.5.0.30/32') }} + {{ peer_block('frontbed', 'ENhRhEgGaFfwV74MqYBHJgkOFpNAF5kVHVK5/tRVTjU=', '10.5.0.17/32') }} + {{ peer_block('garage', 'kFMtVafPU8kJHYmdafc1g/OLRnNPQMGpYKcDqQ9rUjA=', '10.5.0.14/32') }} {{ peer_block('prime', 'vR9lfsUSOIMxkY/k2gRJ6E8ZudccfPpVhrbE9zuxalU=', '10.5.0.0/24', 'public.bigasterisk.com:1195', 50) }} - {{ peer_block('slash', 'IRLLt2yFuXVJbpevAj9d84mGAvi6SbJr1AwLAK/pBTM=', '10.5.0.6/32', 'slash:1195') }} - {{ peer_block('pipe', 'yI0zt8/+baHjadhiBCX6u8sSkhjoh/Q5cIZkGf1H6S4=', '10.5.0.3/32', 'pipe:1195') }} + {{ peer_block('slash', 'IRLLt2yFuXVJbpevAj9d84mGAvi6SbJr1AwLAK/pBTM=', '10.5.0.6/32') }} + {{ peer_block('pipe', 'yI0zt8/+baHjadhiBCX6u8sSkhjoh/Q5cIZkGf1H6S4=', '10.5.0.3/32') }} + {{ peer_block('plus', 'tH2og4BbXaH6BrHSBd73Fx1XT0DxR8vjQxjqHFa913A=', '10.5.0.110/32') }} {% elif host.name == 'prime' %} - {{ peer_block('bang', 'pAxirNVF08R6zYyudhTKjZ9fqC9UKMxknfLi5A39QVY=', '10.5.0.0/24') }} - {{ peer_block('plus', 'tH2og4BbXaH6BrHSBd73Fx1XT0DxR8vjQxjqHFa913A=', '10.5.0.110/32') }} + {{ peer_block('bang', 'xDkAqfljmeVj7bB6VslxD/vVwlUh/vLXX5Wo7ZCoTQ4=', '10.5.0.0/24') }} {{ peer_block('drew-note10', 'QMgx4cmuUTfJ7RH4Q46b54tSQl4eISOmdEney17fnE8=', '10.5.0.112/32') }} {% elif host.name == 'plus' %} - {{ peer_block('bang', 'pAxirNVF08R6zYyudhTKjZ9fqC9UKMxknfLi5A39QVY=', '10.5.0.0/24', '10.2.0.1:1195', 50) }} + {{ peer_block('bang', 'xDkAqfljmeVj7bB6VslxD/vVwlUh/vLXX5Wo7ZCoTQ4=', '10.5.0.0/24', '10.2.0.1:1195', 50) }} {% else %} # I see bang at 10.2.0.1 - {{ peer_block('bang', 'pAxirNVF08R6zYyudhTKjZ9fqC9UKMxknfLi5A39QVY=', '10.5.0.0/24', '10.2.0.1:1195', 50) }} + {{ peer_block('bang', 'xDkAqfljmeVj7bB6VslxD/vVwlUh/vLXX5Wo7ZCoTQ4=', '10.5.0.0/24', '10.2.0.1:1195', 50) }} {% endif %}
--- a/wireguard.py Sun Jul 10 19:50:52 2022 -0700 +++ b/wireguard.py Sun Jul 10 19:51:16 2022 -0700 @@ -12,6 +12,10 @@ def peer_block(hostname, public_key, allowed_ips, endpoint=None, keepalive=None): + # if allowed_ips.startswith('10.5'): + # # k3s nets also need to travel over wg + # allowed_ips += ', 10.42.0.0/24, 10.43.0.0/24' + out = f'''\ [Peer] @@ -33,7 +37,6 @@ # note- this is specific to the wg0 setup. Other conf files don't use it. wireguard_ip = host.host_data['wireguard_address'] - apt.packages(packages=['wireguard']) # new pi may fail with 'Unable to access interface: Protocol not supported'. reboot fixes. priv_key_lines = host.get_fact(FindInFile, path=f'/etc/wireguard/{wireguard_interface}.conf', pattern=r'PrivateKey.*') @@ -65,7 +68,4 @@ systemd.service(service=svc, daemon_reload=True, restarted=True, enabled=True) if host.name == 'bang': - # recompute, or else maybe dnsmasq_10.5 won't start - server.shell("systemctl enable dnsmasq_10.2.service") - server.shell("systemctl enable dnsmasq_10.5.service") - server.shell("systemctl enable wg-quick@wg0.service") \ No newline at end of file + systemd.service(service=f'dnsmasq_10.5', enabled=True, restarted=True, daemon_reload=True)