--- a/kube.py	Sun Jul 10 19:50:52 2022 -0700
+++ b/kube.py	Sun Jul 10 19:51:16 2022 -0700
@@ -1,3 +1,4 @@
+import os
 import tempfile
 from pyinfra import host
 from pyinfra.facts.files import FindInFile
@@ -5,19 +6,19 @@
 from pyinfra.operations import files, server, systemd
 
 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
-
+raise NotImplementedError("update templates from current config files")
 # https://github.com/k3s-io/k3s/releases
-k3s_version = 'v1.24.2-rc1+k3s1'
+# 1.23.6 per https://github.com/cilium/cilium/issues/20331
+k3s_version = 'v1.23.6+k3s1'
 
 # https://github.com/GoogleContainerTools/skaffold/releases
-skaffold_version = 'v1.39.0'
+skaffold_version = 'v1.39.1'
 
 master_ip = "10.5.0.1"
 server_node = 'bang'
-nodes = ['slash', 'dash', 'frontbed', 'garage']
+nodes = ['slash', 'dash']  #, 'dash', 'frontbed', 'garage']
 admin_from = ['bang', 'slash', 'dash']
-
-if host.name in nodes + [server_node]:
+def host_prep():
     server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
     server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
     server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
@@ -41,21 +42,30 @@
             files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
             # pi needs reboot now
 
+    # https://github.com/k3s-io/k3s/issues/1812 unclear
     server.shell(commands=[
-        'update-alternatives --set iptables /usr/sbin/iptables-nft',
-        'update-alternatives --set ip6tables /usr/sbin/ip6tables-nft',
+        'update-alternatives --set iptables /usr/sbin/iptables-legacy',
+        'update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy',
     ])
     # needs reboot if this changed
 
     # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
     files.directory(path='/etc/rancher/k3s')
-    files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
+
+def config_and_run_service():
+    service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
+    which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2'
+    role = 'server' if host.name == server_node else 'agent'
 
-    service_name = 'k3s.service' if host.name == 'bang' else 'k3s-node.service'
-    which_conf = 'config.yaml.j2' if host.name == 'bang' else 'node-config.yaml.j2'
-
-    # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token
-    token = open('secrets/k3s_token', 'rt').read().strip()
+    # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
+    # so this presumes a previous run
+    if host.name == server_node:
+        token="ununsed"
+    else:
+        if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
+            print("first pass is for server only- skipping other nodes")
+            return
+        token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip()
     files.template(
         src=f'templates/kube/{which_conf}',
         dest='/etc/k3s_config.yaml',
@@ -63,29 +73,25 @@
         token=token,
         wg_ip=host.host_data['wireguard_address'],
     )
-    files.template(
-        src='templates/kube/k3s_resolv.conf.j2',
-        dest='/etc/k3s_resolv.conf',
-        master_ip=master_ip,
-        wg_ip=host.host_data['wireguard_address'],
-    )
-    files.template(
-        src='templates/kube/k3s_flannel.conf.j2',
-        dest='/etc/k3s_flannel.conf',
-        master_ip=master_ip,
-        wg_ip=host.host_data['wireguard_address'],
-    )
-    files.put(
-        src='templates/kube/flannel.link',  #
-        dest='/etc/systemd/network/10-flannel.link')  # then reboot
+    # files.put(
+    #     src='templates/kube/flannel.link',  #
+    #     dest='/etc/systemd/network/10-flannel.link')  # then reboot
     files.template(
         src='templates/kube/k3s.service.j2',
         dest=f'/etc/systemd/system/{service_name}',
-        role='server' if host.name == 'bang' else 'agent',
+        role=role,
     )
     systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
 
-if host.name == 'bang':
+
+if host.name in nodes + [server_node]:
+    host_prep()
+
+    # not until registry is up, right?
+    files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
+    config_and_run_service()
+
+if host.name == server_node:
     files.put(
         src="templates/kube/coredns.yaml",
         dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml",
@@ -121,8 +127,8 @@
     files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
     files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
 
-    files.chown(target='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp')
-    files.chmod(target='/etc/rancher/k3s/k3s.yaml', mode='640')
+    # assumes pyinfra is running on server_node
+    files.put(src='/etc/rancher/k3s/k3s.yaml', dest='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640')
 
     # see https://github.com/GoogleContainerTools/skaffold/releases
     files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
@@ -131,3 +137,5 @@
                    group='root',
                    mode='755',
                    cache_time=1000)
+    # one time; writes to $HOME
+    #skaffold config set --global insecure-registries bang5:5000
\ No newline at end of file