Mercurial > code > home > repos > infra

annotate kube.py @ 107:d70816c7c7db

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix pipe net forwarding after update to ubuntu 22.04
author drewp
date Tue, 19 Jul 2022 17:40:02 -0700
parents 8b8ef9d8f0fd
children 301869fa0ed6
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
1 import os
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
2
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
3 from pyinfra import host
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
4 from pyinfra.facts.files import FindInFile
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
5 from pyinfra.facts.server import Arch, LinuxDistribution
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
6 from pyinfra.operations import files, server, systemd
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
7
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
8 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
9
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
10 master_ip = "10.5.0.1"
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
11 server_node = 'bang'
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
12 nodes = ['slash', 'dash'] #, 'dash', 'frontbed', 'garage']
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
13 admin_from = ['bang', 'slash', 'dash']
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
14 # https://github.com/k3s-io/k3s/releases
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
15 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
16 k3s_version = 'v1.23.6+k3s1'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
17
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
18 # https://github.com/GoogleContainerTools/skaffold/releases
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
19 skaffold_version = 'v1.39.1'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
20
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
21
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
22 def download_k3s():
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
23 tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
24 files.download(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
25 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
26 dest='/usr/local/bin/k3s',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
27 user='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
28 group='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
29 mode='755',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
30 cache_time=43000,
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
31 #force=True, # to get a new version
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
32 )
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
33
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
34
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
35 def install_skaffold():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
36 files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
37 dest='/usr/local/bin/skaffold',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
38 user='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
39 group='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
40 mode='755',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
41 cache_time=1000)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
42 # one time; writes to $HOME
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
43 #skaffold config set --global insecure-registries bang5:5000
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
44
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
45
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
46 def pi_cgroup_setup():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
47 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
48 if 'cgroup' not in old_cmdline:
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
49 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
50 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
51 # pi needs reboot now
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
52
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
53
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
54 def host_prep():
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
55 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
56 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
57 server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
58 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
59
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
60 # https://sysctl-explorer.net/net/ipv4/rp_filter/
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
61 none, strict, loose = 0, 1, 2
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
62 server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
63
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
64 if is_pi:
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
65 pi_cgroup_setup()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
66
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
67
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
68 def config_and_run_service():
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
69 download_k3s()
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
70 service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
71 role = 'server' if host.name == server_node else 'agent'
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
72 which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
73
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
74 # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
75 # so this presumes a previous run
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
76 if host.name == server_node:
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
77 token = "ununsed"
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
78 else:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
79 if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
80 print("first pass is for server only- skipping other nodes")
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
81 return
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
82 token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
83 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
84 src=f'templates/kube/{which_conf}',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
85 dest='/etc/k3s_config.yaml',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
86 master_ip=master_ip,
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
87 token=token,
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
88 wg_ip=host.host_data['wireguard_address'],
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
89 )
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
90 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
91 src='templates/kube/k3s.service.j2',
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
92 dest=f'/etc/systemd/system/{service_name}',
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
93 role=role,
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
94 )
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
95 systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
96
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
97
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
98 if host.name in nodes + [server_node]:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
99 host_prep()
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
100 files.directory(path='/etc/rancher/k3s')
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
101
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
102 # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
103 # user confusions: https://github.com/rancher/k3s/issues/1802
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
104 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
107
d70816c7c7db fix pipe net forwarding after update to ubuntu 22.04
drewp
parents: 103
diff changeset
105 # also note that podman dropped the default `docker.io/` prefix on image names (see https://unix.stackexchange.com/a/701785/419418)
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
106 config_and_run_service()
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
107
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
108 if host.name in admin_from:
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
109 files.directory(path='/etc/rancher/k3s')
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
110 install_skaffold()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
111 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
112 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
113 files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
114
103
8b8ef9d8f0fd dead code and templates, reformat, maybe a little refactor
drewp@bigasterisk.com
parents: 99
diff changeset
115 # assumes our pyinfra process is running on server_node
99
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
116 files.put(
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
117 src='/etc/rancher/k3s/k3s.yaml',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
118 dest='/etc/rancher/k3s/k3s.yaml', #
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
119 user='root',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
120 group='drewp',
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
121 mode='640')
6e159d3bdd40 rewrite k3s to match current config. many tests lying around in comments.
drewp@bigasterisk.com
parents: 89
diff changeset
122 server.shell(f"kubectl config set-cluster default --server=https://{master_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")