Mercurial > code > home > repos > infra

annotate kube.py @ 89:2fddde57231b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
no connman to surprisingly rewrite net configs
author drewp@bigasterisk.com
date Sun, 10 Jul 2022 19:51:16 -0700
parents eb38553a6806
children 6e159d3bdd40
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
1 import os
61
b46df76991b6 10.1 cleanups; verbose settings; address updates
drewp@bigasterisk.com
parents: 58
diff changeset
2 import tempfile
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
3 from pyinfra import host
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
4 from pyinfra.facts.files import FindInFile
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
5 from pyinfra.facts.server import Arch, LinuxDistribution
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
6 from pyinfra.operations import files, server, systemd
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
7
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
8 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
9 raise NotImplementedError("update templates from current config files")
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
10 # https://github.com/k3s-io/k3s/releases
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
11 # 1.23.6 per https://github.com/cilium/cilium/issues/20331
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
12 k3s_version = 'v1.23.6+k3s1'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
13
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
14 # https://github.com/GoogleContainerTools/skaffold/releases
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
15 skaffold_version = 'v1.39.1'
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
16
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
17 master_ip = "10.5.0.1"
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
18 server_node = 'bang'
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
19 nodes = ['slash', 'dash'] #, 'dash', 'frontbed', 'garage']
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
20 admin_from = ['bang', 'slash', 'dash']
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
21 def host_prep():
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
22 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
23 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
72
f0e59adf7b91 updates that aren't pkg or version changes
drewp@bigasterisk.com
parents: 67
diff changeset
24 server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
f0e59adf7b91 updates that aren't pkg or version changes
drewp@bigasterisk.com
parents: 67
diff changeset
25 server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
26
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
27 tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
28 files.download(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
29 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
30 dest='/usr/local/bin/k3s',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
31 user='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
32 group='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
33 mode='755',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
34 cache_time=43000,
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
35 #force=True, # to get a new version
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
36 )
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
37
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
38 if is_pi:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
39 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
40 if 'cgroup' not in old_cmdline:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
41 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
42 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
43 # pi needs reboot now
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
44
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
45 # https://github.com/k3s-io/k3s/issues/1812 unclear
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
46 server.shell(commands=[
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
47 'update-alternatives --set iptables /usr/sbin/iptables-legacy',
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
48 'update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy',
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
49 ])
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
50 # needs reboot if this changed
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
51
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
52 # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
53 files.directory(path='/etc/rancher/k3s')
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
54
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
55 def config_and_run_service():
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
56 service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
57 which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2'
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
58 role = 'server' if host.name == server_node else 'agent'
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
59
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
60 # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
61 # so this presumes a previous run
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
62 if host.name == server_node:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
63 token="ununsed"
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
64 else:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
65 if not os.path.exists('/var/lib/rancher/k3s/server/node-token'):
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
66 print("first pass is for server only- skipping other nodes")
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
67 return
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
68 token = open('/var/lib/rancher/k3s/server/node-token', 'rt').read().strip()
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
69 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
70 src=f'templates/kube/{which_conf}',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
71 dest='/etc/k3s_config.yaml',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
72 master_ip=master_ip,
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
73 token=token,
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
74 wg_ip=host.host_data['wireguard_address'],
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
75 )
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
76 # files.put(
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
77 # src='templates/kube/flannel.link', #
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
78 # dest='/etc/systemd/network/10-flannel.link') # then reboot
84
eb38553a6806 trying to fix k3s networking but this doesn't work yet
drewp@bigasterisk.com
parents: 80
diff changeset
79 files.template(
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
80 src='templates/kube/k3s.service.j2',
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
81 dest=f'/etc/systemd/system/{service_name}',
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
82 role=role,
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
83 )
28
d844cdf86164 more old changes, not sure
drewp@bigasterisk.com
parents: 27
diff changeset
84 systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
85
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
86
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
87 if host.name in nodes + [server_node]:
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
88 host_prep()
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
90 # not until registry is up, right?
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
91 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
92 config_and_run_service()
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
93
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
94 if host.name == server_node:
58
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
95 files.put(
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
96 src="templates/kube/coredns.yaml",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
97 dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
98 mode="600",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
99 )
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
100 # files.put(
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
101 # src="templates/kube/coredns-map.yaml",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
102 # dest="/var/lib/rancher/k3s/server/manifests/coredns-map.yaml",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
103 # mode="600",
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
104 # )
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
105 # tmp = tempfile.NamedTemporaryFile(suffix='.yaml')
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
106 # files.template(
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
107 # src='templates/kube/Corefile.yaml.j2',
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
108 # dest=tmp.name,
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
109 # )
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
110 # server.shell(commands=[
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
111 # 'kubectl replace configmap '
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
112 # # '-n kube-system '
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
113 # # 'coredns '
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
114 # f'--filename={tmp.name} '
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
115 # '-o yaml '
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
116 # # '--dry-run=client | kubectl apply -',
f39ada0b8827 push a coredns config with the right forwarding server
drewp@bigasterisk.com
parents: 56
diff changeset
117 # ])
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
118 # one-time thing at cluster create time? not sure
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
119 # - name: Replace https://localhost:6443 by https://master-ip:6443
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
120 # command: >-
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
121 # k3s kubectl config set-cluster default
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
122 # --server=https://{{ master_ip }}:6443
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
123 # --kubeconfig ~{{ ansible_user }}/.kube/config
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
124
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
125 if host.name in admin_from:
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
126 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
127 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
128 files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
129
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
130 # assumes pyinfra is running on server_node
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
131 files.put(src='/etc/rancher/k3s/k3s.yaml', dest='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640')
17
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
132
29
911da5e10834 bump k3s and skaffold
drewp@bigasterisk.com
parents: 28
diff changeset
133 # see https://github.com/GoogleContainerTools/skaffold/releases
40
193cd3b051c6 versions and ips
drewp@bigasterisk.com
parents: 29
diff changeset
134 files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
17
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
135 dest='/usr/local/bin/skaffold',
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
136 user='root',
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
137 group='root',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
138 mode='755',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
139 cache_time=1000)
89
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
140 # one time; writes to $HOME
2fddde57231b no connman to surprisingly rewrite net configs
drewp@bigasterisk.com
parents: 84
diff changeset
141 #skaffold config set --global insecure-registries bang5:5000