Mercurial > code > home > repos > infra
diff kube.py @ 99:6e159d3bdd40
rewrite k3s to match current config. many tests lying around in comments.
author | drewp@bigasterisk.com |
---|---|
date | Fri, 15 Jul 2022 14:37:12 -0700 |
parents | 2fddde57231b |
children | 8b8ef9d8f0fd |
line wrap: on
line diff
--- a/kube.py Fri Jul 15 14:36:00 2022 -0700 +++ b/kube.py Fri Jul 15 14:37:12 2022 -0700 @@ -6,7 +6,11 @@ from pyinfra.operations import files, server, systemd is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux'] -raise NotImplementedError("update templates from current config files") + +master_ip = "10.5.0.1" +server_node = 'bang' +nodes = ['slash', 'dash'] #, 'dash', 'frontbed', 'garage'] +admin_from = ['bang', 'slash', 'dash'] # https://github.com/k3s-io/k3s/releases # 1.23.6 per https://github.com/cilium/cilium/issues/20331 k3s_version = 'v1.23.6+k3s1' @@ -14,16 +18,8 @@ # https://github.com/GoogleContainerTools/skaffold/releases skaffold_version = 'v1.39.1' -master_ip = "10.5.0.1" -server_node = 'bang' -nodes = ['slash', 'dash'] #, 'dash', 'frontbed', 'garage'] -admin_from = ['bang', 'slash', 'dash'] -def host_prep(): - server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True) - server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True) - server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True) - server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True) +def download_k3s(): tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf' files.download( src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}', @@ -35,12 +31,38 @@ #force=True, # to get a new version ) + +def install_skaffold(): + files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64', + dest='/usr/local/bin/skaffold', + user='root', + group='root', + mode='755', + cache_time=1000) + # one time; writes to $HOME + #skaffold config set --global insecure-registries bang5:5000 + + +def pi_cgroup_setup(): + old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0] + if 'cgroup' not in old_cmdline: + cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline) + # pi needs reboot now + + +def host_prep(): + server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True) + server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True) + server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True) + server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True) + + # https://sysctl-explorer.net/net/ipv4/rp_filter/ + none, strict, loose = 0, 1, 2 + server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True) + if is_pi: - old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0] - if 'cgroup' not in old_cmdline: - cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' - files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline) - # pi needs reboot now + pi_cgroup_setup() # https://github.com/k3s-io/k3s/issues/1812 unclear server.shell(commands=[ @@ -49,10 +71,9 @@ ]) # needs reboot if this changed - # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/ - files.directory(path='/etc/rancher/k3s') def config_and_run_service(): + download_k3s() service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service' which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2' role = 'server' if host.name == server_node else 'agent' @@ -60,7 +81,7 @@ # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token, # so this presumes a previous run if host.name == server_node: - token="ununsed" + token = "ununsed" else: if not os.path.exists('/var/lib/rancher/k3s/server/node-token'): print("first pass is for server only- skipping other nodes") @@ -84,6 +105,9 @@ systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True) +# See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/ +files.directory(path='/etc/rancher/k3s') + if host.name in nodes + [server_node]: host_prep() @@ -91,12 +115,12 @@ files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml') config_and_run_service() -if host.name == server_node: - files.put( - src="templates/kube/coredns.yaml", - dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml", - mode="600", - ) +# if host.name == server_node: +# files.put( +# src="templates/kube/coredns.yaml", +# dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml", +# mode="600", +# ) # files.put( # src="templates/kube/coredns-map.yaml", # dest="/var/lib/rancher/k3s/server/manifests/coredns-map.yaml", @@ -115,27 +139,18 @@ # '-o yaml ' # # '--dry-run=client | kubectl apply -', # ]) -# one-time thing at cluster create time? not sure -# - name: Replace https://localhost:6443 by https://master-ip:6443 -# command: >- -# k3s kubectl config set-cluster default -# --server=https://{{ master_ip }}:6443 -# --kubeconfig ~{{ ansible_user }}/.kube/config if host.name in admin_from: + install_skaffold() files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s') files.directory(path='/home/drewp/.kube', user='drewp', group='drewp') files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml') # assumes pyinfra is running on server_node - files.put(src='/etc/rancher/k3s/k3s.yaml', dest='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640') - - # see https://github.com/GoogleContainerTools/skaffold/releases - files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64', - dest='/usr/local/bin/skaffold', - user='root', - group='root', - mode='755', - cache_time=1000) - # one time; writes to $HOME - #skaffold config set --global insecure-registries bang5:5000 \ No newline at end of file + files.put( + src='/etc/rancher/k3s/k3s.yaml', + dest='/etc/rancher/k3s/k3s.yaml', # + user='root', + group='drewp', + mode='640') + server.shell(f"kubectl config set-cluster default --server=https://{master_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")