Mercurial > code > home > repos > infra
annotate multikube.py @ 288:3af02e24eaf9
minor
| author | drewp@bigasterisk.com |
|---|---|
| date | Sun, 21 Apr 2024 17:01:13 -0700 |
| parents | 375515ab60ba |
| children | 65e28d2e0cd8 |
| rev | line source |
|---|---|
| 111 | 1 # leave kube.py running single-host and try again |
| 2 from pyinfra import host | |
| 288 | 3 from pyinfra.facts.server import Arch |
| 111 | 4 from pyinfra.operations import files, server, systemd |
| 5 | |
| 288 | 6 from multikube_config import ( |
| 7 admin_from, | |
| 8 k3s_version, | |
| 9 nodes, | |
| 10 server_ip, | |
| 11 server_node, | |
| 12 skaffold_version, | |
| 13 ) | |
| 111 | 14 |
| 15 | |
|
136
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
16 def install_k3s(): |
| 111 | 17 tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf' |
| 18 files.download( | |
| 19 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}', | |
| 20 dest='/usr/local/bin/k3s', | |
| 21 user='root', | |
| 22 group='root', | |
| 23 mode='755', | |
| 24 cache_time=43000, | |
| 25 #force=True, # to get a new version | |
| 26 ) | |
| 27 | |
| 288 | 28 |
|
136
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
29 def install_k3sup(): |
| 288 | 30 files.download(src='https://github.com/alexellis/k3sup/releases/download/0.12.0/k3sup', |
| 31 dest='/usr/local/bin/k3sup', | |
| 32 mode='755') | |
|
136
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
33 # then do like: |
|
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
34 # root@slash:/home/drewp# ./k3sup install --ip 10.2.0.84 --k3s-extra-args '--no-deploy traefik' --ssh-key /root/.ssh/id_ecdsa |
|
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
35 # root@slash:/home/drewp# ./k3sup join --ip 10.2.0.23 --server-ip 10.2.0.84 --ssh-key /root/.ssh/id_ecdsa |
|
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
36 # but it doesn't seem to make networking go any better |
| 111 | 37 |
| 288 | 38 |
| 111 | 39 def install_skaffold(): |
| 40 files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64', | |
| 41 dest='/usr/local/bin/skaffold', | |
| 42 user='root', | |
| 43 group='root', | |
| 44 mode='755', | |
| 45 cache_time=1000) | |
| 46 # one time; writes to $HOME | |
| 47 #skaffold config set --global insecure-registries bang5:5000 | |
| 48 | |
| 49 | |
| 50 def pi_cgroup_setup(): | |
| 51 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0] | |
| 52 if 'cgroup' not in old_cmdline: | |
| 53 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' | |
| 54 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline) | |
| 55 # pi needs reboot now | |
| 56 | |
| 57 | |
| 58 def host_prep(): | |
| 59 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True) | |
| 60 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True) | |
| 288 | 61 server.sysctl(key='net.ipv6.conf.all.disable_ipv6', value='1', persist=True) |
| 111 | 62 server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True) |
| 63 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True) | |
| 64 | |
| 65 # https://sysctl-explorer.net/net/ipv4/rp_filter/ | |
| 66 #none, strict, loose = 0, 1, 2 | |
| 67 #server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True) | |
| 68 | |
| 69 if is_pi: | |
| 70 pi_cgroup_setup() | |
| 71 | |
| 72 def service_name(): | |
| 73 return 'k3s.service' if host.name == server_node else 'k3s-node.service' | |
| 74 | |
| 75 def config_and_run_service(): | |
| 76 role = 'server' if host.name == server_node else 'agent' | |
| 77 which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2' | |
| 78 | |
| 79 if host.name == server_node: | |
| 80 token = "unused" | |
| 81 else: | |
| 82 token = open('/tmp/k3s-token', 'rt').read().strip() | |
| 83 files.template( | |
| 84 src=f'templates/kube/{which_conf}', | |
| 85 dest='/etc/k3s_config.yaml', | |
| 112 | 86 server_ip=server_ip, |
| 111 | 87 token=token, |
| 288 | 88 wg_ip=host.host_data['mk_addr'], #wireguard_address'], |
| 111 | 89 ) |
| 90 | |
| 91 files.template( | |
| 92 src='templates/kube/k3s.service.j2', | |
| 93 dest=f'/etc/systemd/system/{service_name()}', | |
| 94 role=role, | |
| 95 ) | |
| 96 systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True) | |
| 97 | |
| 98 if host.name == server_node: | |
| 288 | 99 files.get(src='/var/lib/rancher/k3s/server/node-token', dest='/tmp/k3s-token') |
| 100 files.get(src='/etc/rancher/k3s/k3s.yaml', dest='/tmp/k3s-yaml') | |
| 111 | 101 |
| 112 | 102 |
| 111 | 103 if host.name in nodes + [server_node]: |
| 104 host_prep() | |
| 105 files.directory(path='/etc/rancher/k3s') | |
|
136
375515ab60ba
multikube experiment junk. do not run
drewp@bigasterisk.com
parents:
113
diff
changeset
|
106 install_k3s() |
| 111 | 107 config_and_run_service() |
| 108 | |
| 109 # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/ | |
| 110 # user confusions: https://github.com/rancher/k3s/issues/1802 | |
| 111 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml') | |
| 112 # for the possible registries update: | |
| 113 systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True) | |
| 114 | |
| 115 if host.name in admin_from: | |
| 116 files.directory(path='/etc/rancher/k3s') | |
| 117 install_skaffold() | |
| 118 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s') | |
| 119 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp') | |
| 120 # .zshrc has: export KUBECONFIG=/etc/rancher/k3s/k3s.yaml | |
| 121 | |
| 122 if host.name != server_node: | |
| 123 files.put(src='/tmp/k3s-yaml', dest='/etc/rancher/k3s/k3s.yaml') | |
| 124 | |
| 125 files.file(path='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640') | |
| 288 | 126 server.shell( |
| 127 commands=f"kubectl config set-cluster default --server=https://{server_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml") |