Mercurial > code > home > repos > infra

comparison kube.py @ 289:65e28d2e0cd8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
move static templates to files/ ; use inventory tags for selecting hosts+features ; other refactors
author drewp@bigasterisk.com
date Sun, 21 Apr 2024 17:07:23 -0700
parents 3af02e24eaf9
children 7f0482453ead
comparison
equal deleted inserted replaced
288:3af02e24eaf9 289:65e28d2e0cd8
2 2
3 from pyinfra import host 3 from pyinfra import host
4 from pyinfra.facts.files import FindInFile 4 from pyinfra.facts.files import FindInFile
5 from pyinfra.facts.server import Arch, LinuxDistribution 5 from pyinfra.facts.server import Arch, LinuxDistribution
6 from pyinfra.operations import files, server, systemd 6 from pyinfra.operations import files, server, systemd
7
8 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
9 7
10 # https://github.com/GoogleContainerTools/skaffold/releases 8 # https://github.com/GoogleContainerTools/skaffold/releases
11 skaffold_version = 'v2.10.1' 9 skaffold_version = 'v2.10.1'
12 10
13 11
32 user='root', 30 user='root',
33 group='root', 31 group='root',
34 mode='755', 32 mode='755',
35 cache_time=1000) 33 cache_time=1000)
36 # one time; writes to $HOME 34 # one time; writes to $HOME
37 server.shell("skaffold config set --global insecure-registries reg:5000") 35 server.shell(commands="skaffold config set --global insecure-registries reg:5000")
38
39
40 def pi_cgroup_setup():
41 '''
42 fixes this:
43
44 Mar 29 23:47:11 ws-printer k3s[5999]: time="2024-03-29T23:47:11-07:00" level=fatal msg="failed to find memory cgroup (v2)"
45 '''
46 return 'cmdline.txt lives on pipe now, not on the pi host'
47 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
48 if 'cgroup' not in old_cmdline:
49 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
50 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
51 # pi needs reboot now
52 36
53 37
54 def host_prep(): 38 def host_prep():
55 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True) 39 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
56 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True) 40 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
58 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True) 42 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
59 43
60 # https://sysctl-explorer.net/net/ipv4/rp_filter/ 44 # https://sysctl-explorer.net/net/ipv4/rp_filter/
61 none, strict, loose = 0, 1, 2 45 none, strict, loose = 0, 1, 2
62 server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True) 46 server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
63
64 if is_pi:
65 pi_cgroup_setup()
66 47
67 48
68 # don't try to get aufs-dkms on rpi-- https://github.com/docker/for-linux/issues/709 49 # don't try to get aufs-dkms on rpi-- https://github.com/docker/for-linux/issues/709
69 def podman_insecure_registry(reg): 50 def podman_insecure_registry(reg):
70 files.template(src='templates/kube/podman_registries.conf.j2', dest='/etc/containers/registries.conf.d/reg.conf', reg=reg) 51 files.template(src='templates/kube/podman_registries.conf.j2', dest='/etc/containers/registries.conf.d/reg.conf', reg=reg)
98 files.template( 79 files.template(
99 src='templates/kube/k3s.service.j2', 80 src='templates/kube/k3s.service.j2',
100 dest=f'/etc/systemd/system/{service_name}', 81 dest=f'/etc/systemd/system/{service_name}',
101 role=role, 82 role=role,
102 ) 83 )
103 if host.name in ['bang', 'garage']: 84 if not host.data.get('gpu'):
104 # no supported gpu 85 # no supported gpu
105 ''' 86 '''
106 kubectl label --overwrite node bang nvidia.com/gpu.deploy.gpu-feature-discovery=false 87 kubectl label --overwrite node bang nvidia.com/gpu.deploy.gpu-feature-discovery=false
107 kubectl label --overwrite node bang nvidia.com/gpu.deploy.container-toolkit=false 88 kubectl label --overwrite node bang nvidia.com/gpu.deploy.container-toolkit=false
108 kubectl label --overwrite node bang nvidia.com/gpu.deploy.dcgm-exporter=false 89 kubectl label --overwrite node bang nvidia.com/gpu.deploy.dcgm-exporter=false
133 114
134 def make_cluster( 115 def make_cluster(
135 server_ip, 116 server_ip,
136 server_node, 117 server_node,
137 nodes, 118 nodes,
138 admin_from,
139 # https://github.com/k3s-io/k3s/releases 119 # https://github.com/k3s-io/k3s/releases
140 # 1.23.6 per https://github.com/cilium/cilium/issues/20331 120 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
141 k3s_version, 121 k3s_version,
142 ): 122 ):
143 123
149 # user confusions: https://github.com/rancher/k3s/issues/1802 129 # user confusions: https://github.com/rancher/k3s/issues/1802
150 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml', reg='reg:5000') 130 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml', reg='reg:5000')
151 # also note that podman dropped the default `docker.io/` prefix on image names (see https://unix.stackexchange.com/a/701785/419418) 131 # also note that podman dropped the default `docker.io/` prefix on image names (see https://unix.stackexchange.com/a/701785/419418)
152 config_and_run_service(k3s_version, server_node, server_ip) 132 config_and_run_service(k3s_version, server_node, server_ip)
153 133
154 if host.name in admin_from: 134 if host.data.get('k8s_admin'):
155 podman_insecure_registry(reg='reg:5000') 135 podman_insecure_registry(reg='reg:5000')
156 files.directory(path='/etc/rancher/k3s') 136 files.directory(path='/etc/rancher/k3s')
157 install_skaffold() 137 install_skaffold()
158 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s') 138 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
159 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp') 139 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')