Mercurial > code > home > repos > infra

annotate kube/kube.py @ 332:d4893670f888 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
WIP: use watchdog reboot timer on pi
author drewp@bigasterisk.com
date Thu, 27 Feb 2025 11:09:29 -0800
parents 5b88b38f2471
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
303
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
1 import io
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
2 import os
303
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
3 import subprocess
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
4 from tempfile import NamedTemporaryFile
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
5
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
6 from pyinfra.context import host
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
7 from pyinfra.facts.files import FindInFile
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
8 from pyinfra.facts.server import Arch, LinuxDistribution
303
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
9 from pyinfra.operations import files, server, systemd, apt
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
10
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
11 # https://github.com/GoogleContainerTools/skaffold/releases
307
89b948759d64 update skaffold
drewp@bigasterisk.com
parents: 303
diff changeset
12 skaffold_version = 'v2.13.2'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
13
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
14
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
15 def download_k3s(k3s_version):
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
16 match host.get_fact(Arch):
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
17 case 'x86_64':
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
18 tail,sha = 'k3s','dd320550cb32b053f78fb6442a1cd2c0188428c5c817482763a7fb32ea3b87b8'
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
19 case 'aarch64':
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
20 tail,sha = 'k3s-arm64','9c4cc7586c4999650edbb5312f114d4e9c6517143b1234206a32464397c77c41'
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
21 case _:
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
22 raise ValueError(f"unknown arch: {host.get_fact(Arch)}")
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
23 files.download(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
24 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
25 dest='/usr/local/bin/k3s',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
26 user='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
27 group='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
28 mode='755',
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
29 sha256sum=sha,
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
30 cache_time=1000,
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
31 )
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
32
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
33
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
34 def install_skaffold(reg):
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
35 files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
36 dest='/usr/local/bin/skaffold',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
37 user='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
38 group='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
39 mode='755',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
40 cache_time=1000)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
41 # one time; writes to $HOME
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
42 server.shell(commands=f"skaffold config set --global insecure-registries {reg}")
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
43
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
44
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
45 def host_prep():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
46 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
47 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
48 server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
49 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
50
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
51 # https://sysctl-explorer.net/net/ipv4/rp_filter/
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
52 none, strict, loose = 0, 1, 2
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
53 server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
54
268
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
55
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
56 # don't try to get aufs-dkms on rpi-- https://github.com/docker/for-linux/issues/709
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
57 def podman_insecure_registry(reg):
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
58 # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
59 # user confusions: https://github.com/rancher/k3s/issues/1802
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
60 files.template(src='kube/templates/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml', reg=reg)
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
61
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
62 files.template(src='kube/templates/podman_registries.conf.j2', dest='/etc/containers/registries.conf.d/reg.conf', reg=reg)
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
63 if host.data.get('k8s_admin'):
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
64 systemd.service(service='podman', user_mode=True)
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
65 systemd.service(service='podman.socket', user_mode=True)
296
drewp@bigasterisk.com
parents: 295
diff changeset
66 # and maybe edit /etc/containers/policy.json
240
b58f05be720a updates for new ubuntu 23.10 install
drewp@bigasterisk.com
parents: 232
diff changeset
67
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
68
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
69 def config_and_run_service(k3s_version, server_node, server_ip):
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
70 download_k3s(k3s_version)
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
71 service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
72 role = 'server' if host.name == server_node else 'agent'
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
73 which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
74
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
75 files.put(src="kube/files/kubelet.config", dest="/etc/rancher/k3s/kubelet.config")
294
7f0482453ead boost pod-per-node limit
drewp@bigasterisk.com
parents: 289
diff changeset
76
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
77 # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
78 # so this presumes a previous run
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
79 if host.name == server_node:
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
80 token = "ununsed"
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
81 else:
267
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
82 # this assumes localhost is the k3s server.
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
83 if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
84 print("first pass is for server only- skipping other nodes")
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
85 return
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
86 token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
87 files.template(
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
88 src=f'kube/templates/{which_conf}',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
89 dest='/etc/k3s_config.yaml',
112
drewp@bigasterisk.com
parents: 107
diff changeset
90 server_ip=server_ip,
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
91 token=token,
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
92 wg_ip=host.host_data['wireguard_address'],
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
93 )
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
94 files.template(
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
95 src='kube/templates/k3s.service.j2',
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
96 dest=f'/etc/systemd/system/{service_name}',
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
97 role=role,
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
98 )
289
65e28d2e0cd8 move static templates to files/ ; use inventory tags for selecting hosts+features ; other refactors
drewp@bigasterisk.com
parents: 288
diff changeset
99 if not host.data.get('gpu'):
268
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
100 # no supported gpu
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
101 '''
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
102 kubectl label --overwrite node bang nvidia.com/gpu.deploy.gpu-feature-discovery=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
103 kubectl label --overwrite node bang nvidia.com/gpu.deploy.container-toolkit=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
104 kubectl label --overwrite node bang nvidia.com/gpu.deploy.dcgm-exporter=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
105 kubectl label --overwrite node bang nvidia.com/gpu.deploy.device-plugin=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
106 kubectl label --overwrite node bang nvidia.com/gpu.deploy.driver=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
107 kubectl label --overwrite node bang nvidia.com/gpu.deploy.mig-manager=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
108 kubectl label --overwrite node bang nvidia.com/gpu.deploy.operator-validator=false
34ab4aec7d4b notes and changes for getting nvidia gpu k3d support going, which was very hard
drewp@bigasterisk.com
parents: 267
diff changeset
109 '''
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
110 systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
111
267
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
112
254
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
113 def setupNvidiaToolkit():
265
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
114 # guides:
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
115 # https://github.com/NVIDIA/k8s-device-plugin#prerequisites
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
116 # https://docs.k3s.io/advanced#nvidia-container-runtime-support
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
117 # apply this once to kube-system: https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.3/nvidia-device-plugin.yml
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
118 # apply this once: https://raw.githubusercontent.com/NVIDIA/gpu-feature-discovery/v0.8.2/deployments/static/nfd.yaml
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
119 # and: kubectl apply -f https://raw.githubusercontent.com/NVIDIA/gpu-feature-discovery/v0.8.2/deployments/static/gpu-feature-discovery-daemonset.yaml
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
120
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
121 # k3s says they do this:
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
122 #server.shell('nvidia-ctk runtime configure --runtime=containerd --config /var/lib/rancher/k3s/agent/etc/containerd/config.toml')
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
123
254
11b738d4c4ae work on k8s/nvidia setup
drewp@bigasterisk.com
parents: 240
diff changeset
124 # then caller restarts k3s which includes containerd
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
125
265
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
126 # tried https://github.com/k3s-io/k3s/discussions/9231#discussioncomment-8114243
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
127 pass
9d0a3915cc00 WIP on k8s+gpu
drewp@bigasterisk.com
parents: 263
diff changeset
128
267
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
129
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
130 def make_cluster(
267
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
131 server_ip,
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
132 server_node,
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
133 nodes,
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
134 # https://github.com/k3s-io/k3s/releases
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
135 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
136 k3s_version,
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
137 ):
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
138 if host.name in nodes + [server_node]:
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
139 host_prep()
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
140 files.directory(path='/etc/rancher/k3s')
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
141
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
142 podman_insecure_registry(reg='reg:5000')
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
143 # also note that podman dropped the default `docker.io/` prefix on image names (see https://unix.stackexchange.com/a/701785/419418)
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
144 config_and_run_service(k3s_version, server_node, server_ip)
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
145
289
65e28d2e0cd8 move static templates to files/ ; use inventory tags for selecting hosts+features ; other refactors
drewp@bigasterisk.com
parents: 288
diff changeset
146 if host.data.get('k8s_admin'):
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
147 files.directory(path='/etc/rancher/k3s')
302
3204157bb3e5 attempt to fix http-registry problem on rpi
drewp@bigasterisk.com
parents: 296
diff changeset
148 install_skaffold("reg:5000")
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
149 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
150 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
151
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
152 # assumes our pyinfra process is running on server_node
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
153 files.put(
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
154 src='/etc/rancher/k3s/k3s.yaml',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
155 dest='/etc/rancher/k3s/k3s.yaml', #
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
156 user='root',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
157 group='drewp',
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
158 mode='640')
267
564b62e59484 redo k3s with ditto as the server
drewp@bigasterisk.com
parents: 265
diff changeset
159 server.shell(
288
drewp@bigasterisk.com
parents: 287
diff changeset
160 commands=f"kubectl config set-cluster default --server=https://{server_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml"
drewp@bigasterisk.com
parents: 287
diff changeset
161 )
204
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
162
ad6e997fd323 prepare for k8s server host and registry host to change
drewp@bigasterisk.com
parents: 203
diff changeset
163
303
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
164 def run_non_k8s_telegraf(node):
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
165 if host.name != node:
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
166 return
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
167 # this CM is written by /my/serv/telegraf/tasks.py
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
168 conf = io.BytesIO(subprocess.check_output(["kubectl", "get", "cm", "telegraf-config", "-o", "jsonpath={.data." + node + "}"]))
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
169 apt.packages(packages=['telegraf'])
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
170 files.put(src=conf, dest="/etc/telegraf/telegraf.conf", create_remote_dir=True, assume_exists=True)
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
171 systemd.service(
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
172 service='telegraf',
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
173 running=True,
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
174 enabled=True,
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
175 restarted=True,
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
176 )
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
177
9e15c07d5258 get telegraf running on pipe (non k8s)
drewp@bigasterisk.com
parents: 302
diff changeset
178
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
179 def main_cluster():
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
180 make_cluster(
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
181 server_ip="10.5.0.7",
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
182 server_node='ditto',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
183 nodes=[
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
184 'bang',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
185 'slash',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
186 'dash',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
187 'ws-printer',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
188 'ga-iot',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
189 'li-drums',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
190 # 'gn-music',
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
191 ],
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
192 k3s_version='v1.29.1+k3s1')
287
c094d23b7943 refmt, notes
drewp@bigasterisk.com
parents: 282
diff changeset
193
326
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
194 run_non_k8s_telegraf('pipe')
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
195
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
196
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
197 operations = [
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
198 main_cluster,
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
199 ]
5b88b38f2471 huge reorg, reog toplevel functions in preparation of a ui with nice task lists
drewp@bigasterisk.com
parents: 316
diff changeset
200
287
c094d23b7943 refmt, notes
drewp@bigasterisk.com
parents: 282
diff changeset
201 # consider https://github.com/derailed/k9s/releases/download/v0.32.4/k9s_Linux_amd64.tar.gz
296
drewp@bigasterisk.com
parents: 295
diff changeset
202
drewp@bigasterisk.com
parents: 295
diff changeset
203 # k label node ws-printer unschedulable=octoprint-allowed