diff kube.py @ 99:6e159d3bdd40

rewrite k3s to match current config. many tests lying around in comments.
author drewp@bigasterisk.com
date Fri, 15 Jul 2022 14:37:12 -0700
parents 2fddde57231b
children 8b8ef9d8f0fd
line wrap: on
line diff
--- a/kube.py	Fri Jul 15 14:36:00 2022 -0700
+++ b/kube.py	Fri Jul 15 14:37:12 2022 -0700
@@ -6,7 +6,11 @@
 from pyinfra.operations import files, server, systemd
 
 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
-raise NotImplementedError("update templates from current config files")
+
+master_ip = "10.5.0.1"
+server_node = 'bang'
+nodes = ['slash', 'dash']  #, 'dash', 'frontbed', 'garage']
+admin_from = ['bang', 'slash', 'dash']
 # https://github.com/k3s-io/k3s/releases
 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
 k3s_version = 'v1.23.6+k3s1'
@@ -14,16 +18,8 @@
 # https://github.com/GoogleContainerTools/skaffold/releases
 skaffold_version = 'v1.39.1'
 
-master_ip = "10.5.0.1"
-server_node = 'bang'
-nodes = ['slash', 'dash']  #, 'dash', 'frontbed', 'garage']
-admin_from = ['bang', 'slash', 'dash']
-def host_prep():
-    server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
-    server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
-    server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
-    server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
 
+def download_k3s():
     tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
     files.download(
         src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
@@ -35,12 +31,38 @@
         #force=True,  # to get a new version
     )
 
+
+def install_skaffold():
+    files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
+                   dest='/usr/local/bin/skaffold',
+                   user='root',
+                   group='root',
+                   mode='755',
+                   cache_time=1000)
+    # one time; writes to $HOME
+    #skaffold config set --global insecure-registries bang5:5000
+
+
+def pi_cgroup_setup():
+    old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
+    if 'cgroup' not in old_cmdline:
+        cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
+        files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
+        # pi needs reboot now
+
+
+def host_prep():
+    server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
+    server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
+    server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
+    server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
+
+    # https://sysctl-explorer.net/net/ipv4/rp_filter/
+    none, strict, loose = 0, 1, 2
+    server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
+
     if is_pi:
-        old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
-        if 'cgroup' not in old_cmdline:
-            cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
-            files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
-            # pi needs reboot now
+        pi_cgroup_setup()
 
     # https://github.com/k3s-io/k3s/issues/1812 unclear
     server.shell(commands=[
@@ -49,10 +71,9 @@
     ])
     # needs reboot if this changed
 
-    # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
-    files.directory(path='/etc/rancher/k3s')
 
 def config_and_run_service():
+    download_k3s()
     service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
     which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2'
     role = 'server' if host.name == server_node else 'agent'
@@ -60,7 +81,7 @@
     # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
     # so this presumes a previous run
     if host.name == server_node:
-        token="ununsed"
+        token = "ununsed"
     else:
         if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
             print("first pass is for server only- skipping other nodes")
@@ -84,6 +105,9 @@
     systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
 
 
+# See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
+files.directory(path='/etc/rancher/k3s')
+
 if host.name in nodes + [server_node]:
     host_prep()
 
@@ -91,12 +115,12 @@
     files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
     config_and_run_service()
 
-if host.name == server_node:
-    files.put(
-        src="templates/kube/coredns.yaml",
-        dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml",
-        mode="600",
-    )
+# if host.name == server_node:
+#     files.put(
+#         src="templates/kube/coredns.yaml",
+#         dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml",
+#         mode="600",
+#     )
     # files.put(
     #     src="templates/kube/coredns-map.yaml",
     #     dest="/var/lib/rancher/k3s/server/manifests/coredns-map.yaml",
@@ -115,27 +139,18 @@
     #     '-o yaml '
     #     # '--dry-run=client | kubectl apply -',
     # ])
-# one-time thing at cluster create time? not sure
-# - name: Replace https://localhost:6443 by https://master-ip:6443
-#   command: >-
-#     k3s kubectl config set-cluster default
-#       --server=https://{{ master_ip }}:6443
-#       --kubeconfig ~{{ ansible_user }}/.kube/config
 
 if host.name in admin_from:
+    install_skaffold()
     files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
     files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
     files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
 
     # assumes pyinfra is running on server_node
-    files.put(src='/etc/rancher/k3s/k3s.yaml', dest='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640')
-
-    # see https://github.com/GoogleContainerTools/skaffold/releases
-    files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
-                   dest='/usr/local/bin/skaffold',
-                   user='root',
-                   group='root',
-                   mode='755',
-                   cache_time=1000)
-    # one time; writes to $HOME
-    #skaffold config set --global insecure-registries bang5:5000
\ No newline at end of file
+    files.put(
+        src='/etc/rancher/k3s/k3s.yaml',
+        dest='/etc/rancher/k3s/k3s.yaml',  #
+        user='root',
+        group='drewp',
+        mode='640')
+    server.shell(f"kubectl config set-cluster default --server=https://{master_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")