Mercurial > code > home > repos > infra

annotate kube.py @ 254:11b738d4c4ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work on k8s/nvidia setup
author drewp@bigasterisk.com
date Thu, 25 Jan 2024 18:50:44 -0800
parents b58f05be720a
children 980d81d5f4bb
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
1 import os
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
2
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
3 from pyinfra import host
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
4 from pyinfra.facts.files import FindInFile
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
5 from pyinfra.facts.server import Arch, LinuxDistribution
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
6 from pyinfra.operations import files, server, systemd
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
7
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
8 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
9
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
10 # https://github.com/GoogleContainerTools/skaffold/releases
232
501d867cdd3e skaffold ver
drewp@bigasterisk.com
parents: 222
diff changeset
11 skaffold_version = 'v2.8.0'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
12
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
13
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
14 def download_k3s(k3s_version):
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
15 tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
16 files.download(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
17 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
18 dest='/usr/local/bin/k3s',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
19 user='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
20 group='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
21 mode='755',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
22 cache_time=43000,
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
23 #force=True, # to get a new version
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
24 )
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
25
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
26
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
27 def install_skaffold():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
28 files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
29 dest='/usr/local/bin/skaffold',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
30 user='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
31 group='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
32 mode='755',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
33 cache_time=1000)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
34 # one time; writes to $HOME
220
416aa647efd9 registry hostname, add garage pi back to k8s
drewp@bigasterisk.com
parents: 204
diff changeset
35 server.shell("skaffold config set --global insecure-registries reg:5000")
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
36
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
37
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
38 def pi_cgroup_setup():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
39 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
40 if 'cgroup' not in old_cmdline:
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
41 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
42 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
43 # pi needs reboot now
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
44
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
45
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
46 def host_prep():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
47 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
48 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
49 server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
50 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
51
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
52 # https://sysctl-explorer.net/net/ipv4/rp_filter/
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
53 none, strict, loose = 0, 1, 2
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
54 server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
55
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
56 if is_pi:
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
57 pi_cgroup_setup()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
58
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
59 # don't try to get aufs-dkms on rpi-- https://github.com/docker/for-linux/issues/709
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
60 def podman_insecure_registry(reg):
222
9dfb0916e850 registry change
drewp@bigasterisk.com
parents: 220
diff changeset
61 files.template(src='templates/kube/podman_registries.conf.j2', dest='/etc/containers/registries.conf.d/reg.conf', reg=reg)
240
b58f05be720a updates for new ubuntu 23.10 install
drewp@bigasterisk.com
parents: 232
diff changeset
62 systemd.service(service='podman', user_mode=True)
b58f05be720a updates for new ubuntu 23.10 install
drewp@bigasterisk.com
parents: 232
diff changeset
63 systemd.service(service='podman.socket', user_mode=True)
b58f05be720a updates for new ubuntu 23.10 install
drewp@bigasterisk.com
parents: 232
diff changeset
64
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
65
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
66 def config_and_run_service(k3s_version, server_node, server_ip):
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
67 download_k3s(k3s_version)
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
68 service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
69 role = 'server' if host.name == server_node else 'agent'
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
70 which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
71
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
72 # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
73 # so this presumes a previous run
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
74 if host.name == server_node:
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
75 token = "ununsed"
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
76 else:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
77 if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
78 print("first pass is for server only- skipping other nodes")
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
79 return
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
80 token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
81 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
82 src=f'templates/kube/{which_conf}',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
83 dest='/etc/k3s_config.yaml',
112
drewp@bigasterisk.com
parents: 107
diff changeset
84 server_ip=server_ip,
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
85 token=token,
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
86 wg_ip=host.host_data['wireguard_address'],
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
87 )
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
88 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
89 src='templates/kube/k3s.service.j2',
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
90 dest=f'/etc/systemd/system/{service_name}',
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
91 role=role,
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
92 )
254
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
93 if host.name in ['dash', 'bang', 'ditto']:
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
94 setupNvidiaToolkit()
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
95 systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
96
254
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
97 def setupNvidiaToolkit():
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
98 server.shell('nvidia-ctk runtime configure --runtime=containerd --config /var/lib/rancher/k3s/agent/etc/containerd/config.toml')
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
99 # then caller restarts k3s which includes containerd
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
100
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
101 def make_cluster(
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
102 server_ip = "10.5.0.1",
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
103 server_node = 'bang',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
104 nodes = ['slash', 'dash', 'ditto'],
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
105 admin_from = ['bang', 'slash', 'dash', 'ditto'],
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
106 # https://github.com/k3s-io/k3s/releases
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
107 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
108 k3s_version = 'v1.24.3+k3s1',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
109 ):
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
110 if not is_pi:
220
416aa647efd9 registry hostname, add garage pi back to k8s
drewp@bigasterisk.com
parents: 204
diff changeset
111 podman_insecure_registry(reg='reg:5000')
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
112
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
113 if host.name in nodes + [server_node]:
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
114 host_prep()
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
115 files.directory(path='/etc/rancher/k3s')
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
116
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
117 # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
118 # user confusions: https://github.com/rancher/k3s/issues/1802
220
416aa647efd9 registry hostname, add garage pi back to k8s
drewp@bigasterisk.com
parents: 204
diff changeset
119 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml', reg='reg:5000')
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
120 # also note that podman dropped the default `docker.io/` prefix on image names (see https://unix.stackexchange.com/a/701785/419418)
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
121 config_and_run_service(k3s_version, server_node, server_ip)
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
122
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
123 if host.name in admin_from:
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
124 files.directory(path='/etc/rancher/k3s')
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
125 install_skaffold()
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
126 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
127 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
128
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
129 # assumes our pyinfra process is running on server_node
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
130 files.put(
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
131 src='/etc/rancher/k3s/k3s.yaml',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
132 dest='/etc/rancher/k3s/k3s.yaml', #
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
133 user='root',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
134 group='drewp',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
135 mode='640')
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
136 server.shell(f"kubectl config set-cluster default --server=https://{server_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
137
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
138
220
416aa647efd9 registry hostname, add garage pi back to k8s
drewp@bigasterisk.com
parents: 204
diff changeset
139 make_cluster( server_ip = "10.5.0.1", server_node = 'bang', nodes = ['slash', 'dash', 'ditto', 'garage'], admin_from = ['bang', 'slash', 'dash', 'ditto'], k3s_version = 'v1.24.3+k3s1')
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
140 #make_cluster( server_ip = "10.5.0.7", server_node = 'ditto', nodes = ['slash', 'dash', 'bang'], admin_from = ['bang', 'slash', 'dash', 'ditto'], k3s_version = 'v1.25.11+k3s1')