Mercurial > code > home > repos > infra

annotate kube.py @ 27:7b22ff272001

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor (may not be a correct commit)
author drewp@bigasterisk.com
date Sat, 15 Jan 2022 14:43:01 -0800
parents 948d9d72267d
children d844cdf86164
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
1 from pyinfra import host
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
2 from pyinfra.facts.files import FindInFile
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
3 from pyinfra.facts.server import Arch, LinuxDistribution
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
4 from pyinfra.operations import files, server, systemd
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
5
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
6 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
7
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
8 k3s_version = 'v1.22.4-rc1+k3s1'
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
9 master_ip = "10.5.0.1"
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
10 server_node = 'bang'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
11 nodes = ['slash', 'dash', 'frontbed', 'garage']
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
12 admin_from = ['bang', 'slash', 'dash']
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
13
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
14 if host.name in [nodes + [server_node]]:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
15 server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
16 server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
17
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
18 tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
19 files.download(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
20 src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
21 dest='/usr/local/bin/k3s',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
22 user='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
23 group='root',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
24 mode='755',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
25 cache_time=43000,
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
26 # force=True, # to get a new version
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
27 )
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
28
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
29 if is_pi:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
30 old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
31 print(repr(old_cmdline))
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
32 if 'cgroup' not in old_cmdline:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
33 cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
34 files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
35 # pi needs reboot now
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
36
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
37 server.shell(commands=[
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
38 'update-alternatives --set iptables /usr/sbin/iptables-nft',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
39 'update-alternatives --set ip6tables /usr/sbin/ip6tables-nft',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
40 ])
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
41 # needs reboot if this changed
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
42
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
43 # See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
44 files.directory(path='/etc/rancher/k3s')
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
45 files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
46
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
47 if host.name == 'bang': # master
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
48 files.template(
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
49 src='templates/kube/config.yaml.j2',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
50 dest='/etc/k3s_config.yaml',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
51 master_ip=master_ip,
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
52 )
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
53 files.template(
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
54 src='templates/kube/Corefile.j2',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
55 dest='/etc/k3s_coredns_config',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
56 )
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
57 files.template(
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
58 src='templates/kube/k3s-server.service.j2',
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
59 dest='/etc/systemd/system/k3s.service',
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
60 master_ip=master_ip,
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
61 )
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
62 systemd.service(service='k3s.service', daemon_reload=True, enabled=True, restarted=True)
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
63
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
64 server.shell(commands=[
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
65 'kubectl replace configmap -n kube-system coredns --from-file=Corefile=/etc/k3s_coredns_config -o yaml --dry-run=client | kubectl apply -',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
66 ])
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
67 # one-time thing at cluster create time? not sure
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
68 # - name: Replace https://localhost:6443 by https://master-ip:6443
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
69 # command: >-
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
70 # k3s kubectl config set-cluster default
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
71 # --server=https://{{ master_ip }}:6443
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
72 # --kubeconfig ~{{ ansible_user }}/.kube/config
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
73
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
74 if host.name in ['slash', 'dash', 'frontbed', 'garage']: # nodes
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
75 # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
76 token = open('secrets/k3s_token', 'rt').read().strip()
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
77
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
78 files.template(
12
15c5ce7c74b5 refactor, cleanup, split large deploys
drewp@bigasterisk.com
parents: 8
diff changeset
79 src='templates/kube/k3s-node.service.j2',
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
80 dest='/etc/systemd/system/k3s-node.service',
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
81 master_ip=master_ip,
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
82 token=token,
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
83 )
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
84
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
85 systemd.service(service='k3s-node.service', daemon_reload=True, enabled=True, restarted=True)
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
86 # if bang:
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
87 # files.template(
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
88 # src='templates/kube/Corefile.j2',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
89 # dest='/etc/k3s_coredns_config',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
90 # )
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
91 # server.shell(commands=[
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
92 # 'kubectl replace configmap '
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
93 # '-n kube-system '
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
94 # 'coredns '
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
95 # '--from-file=Corefile=/etc/k3s_coredns_config '
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
96 # '-o yaml '
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
97 # '--dry-run=client | kubectl apply -',
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
98 # ])
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
99 # one-time thing at cluster create time? not sure
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
100 # - name: Replace https://localhost:6443 by https://master-ip:6443
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
101 # command: >-
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
102 # k3s kubectl config set-cluster default
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
103 # --server=https://{{ master_ip }}:6443
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
104 # --kubeconfig ~{{ ansible_user }}/.kube/config
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
105
27
7b22ff272001 refactor (may not be a correct commit)
drewp@bigasterisk.com
parents: 21
diff changeset
106 if host.name in admin_from:
8
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
107 files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
108 files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
109 files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
110
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
111 files.chown(target='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp')
226f3c8419b2 kube setup
drewp@bigasterisk.com
parents:
diff changeset
112 files.chmod(target='/etc/rancher/k3s/k3s.yaml', mode='640')
17
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
113
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
114 files.download(src='https://storage.googleapis.com/skaffold/releases/v1.35.0/skaffold-linux-amd64',
17
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
115 dest='/usr/local/bin/skaffold',
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
116 user='root',
0c1496e11b8f get skaffold on hosts that want to do deploys
drewp@bigasterisk.com
parents: 12
diff changeset
117 group='root',
21
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
118 mode='755',
948d9d72267d k3s update and some config refactoring
drewp@bigasterisk.com
parents: 19
diff changeset
119 cache_time=1000)