Mercurial > code > home > repos > infra

changeset 111:340d778a1682 main

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
start multikube experiment
author drewp@bigasterisk.com
date Sat, 20 Aug 2022 14:12:05 -0700
parents de407da9224a
children 301869fa0ed6
files files/kube/k3s-killall.sh files/kube/k3s-uninstall.sh inventory.py multikube.py multikube_config.py multikube_wipe.py tasks.py
diffstat 7 files changed, 290 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/files/kube/k3s-killall.sh	Sat Aug 20 14:12:05 2022 -0700
@@ -0,0 +1,76 @@
+#!/bin/sh
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x $service ] && $service stop
+done
+
+pschildren() {
+    ps -e -o ppid= -o pid= | \
+    sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+    grep -w "^$1" | \
+    cut -f2
+}
+
+pstree() {
+    for pid in $@; do
+        echo $pid
+        for child in $(pschildren $pid); do
+            pstree $child
+        done
+    done
+}
+
+killtree() {
+    kill -9 $(
+        { set +x; } 2>/dev/null;
+        pstree $@;
+        set -x;
+    ) 2>/dev/null
+}
+
+getshims() {
+    ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
+}
+
+killtree $({ set +x; } 2>/dev/null; getshims; set -x)
+
+do_unmount_and_remove() {
+    set +x
+    while read -r _ path _; do
+        case "$path" in $1*) echo "$path" ;; esac
+    done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"'
+    set -x
+}
+
+do_unmount_and_remove '/run/k3s'
+do_unmount_and_remove '/var/lib/rancher/k3s'
+do_unmount_and_remove '/var/lib/kubelet/pods'
+do_unmount_and_remove '/var/lib/kubelet/plugins'
+do_unmount_and_remove '/run/netns/cni-'
+
+# Remove CNI namespaces
+ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
+
+# Delete network interface(s) that match 'master cni0'
+ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+    iface=${iface%%@*}
+    [ -z "$iface" ] || ip link delete $iface
+done
+ip link delete cni0
+ip link delete flannel.1
+ip link delete flannel-v6.1
+ip link delete kube-ipvs0
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | grep -v flannel | iptables-restore
+ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -v flannel | ip6tables-restore
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/files/kube/k3s-uninstall.sh	Sat Aug 20 14:12:05 2022 -0700
@@ -0,0 +1,53 @@
+#!/bin/sh
+set -x
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+/usr/local/bin/k3s-killall.sh
+
+if command -v systemctl; then
+    systemctl disable k3s
+    systemctl reset-failed k3s
+    systemctl daemon-reload
+fi
+if command -v rc-update; then
+    rc-update delete k3s default
+fi
+
+rm -f /etc/systemd/system/k3s.service
+rm -f /etc/systemd/system/k3s.service.env
+
+remove_uninstall() {
+    rm -f /usr/local/bin/k3s-uninstall.sh
+}
+trap remove_uninstall EXIT
+
+if (ls /etc/systemd/system/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then
+    set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x
+    exit
+fi
+
+for cmd in kubectl crictl ctr; do
+    if [ -L /usr/local/bin/$cmd ]; then
+        rm -f /usr/local/bin/$cmd
+    fi
+done
+
+rm -rf /etc/rancher/k3s
+rm -rf /run/k3s
+rm -rf /run/flannel
+rm -rf /var/lib/rancher/k3s
+rm -rf /var/lib/kubelet
+rm -f /usr/local/bin/k3s
+rm -f /usr/local/bin/k3s-killall.sh
+
+if type yum >/dev/null 2>&1; then
+    yum remove -y k3s-selinux
+    rm -f /etc/yum.repos.d/rancher-k3s-common*.repo
+elif type zypper >/dev/null 2>&1; then
+    uninstall_cmd="zypper remove -y k3s-selinux"
+    if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then
+        uninstall_cmd="transactional-update --no-selfupdate -d run $uninstall_cmd"
+    fi
+    $uninstall_cmd
+    rm -f /etc/zypp/repos.d/rancher-k3s-common*.repo
+fi
--- a/inventory.py	Sat Aug 20 14:13:02 2022 -0700
+++ b/inventory.py	Sat Aug 20 14:12:05 2022 -0700
@@ -8,13 +8,15 @@
     ('dash', {
         'ssh_hostname': '10.2.0.23',
         'wireguard_address': '10.5.0.5',
+        'mk_addr': '10.2.0.23',
     }),
     ('slash', {
-        'ssh_hostname': '10.2.0.82',
+        'ssh_hostname': 'slash',
         'wireguard_address': '10.5.0.6',
+        'mk_addr': '10.2.0.84',
     }),
     ('dot', {
-        'ssh_hostname': '10.2.0.47',
+        'ssh_hostname': '10.2.0.71',
         'wireguard_address': '10.5.0.30',
     })
 ]
@@ -27,16 +29,16 @@
 ]
 
 pi = [
-   ('frontbed', {
-       'mac': 'b8:27:eb:e9:d3:44',
-       'ssh_hostname': '10.2.0.27',
-       'wireguard_address': '10.5.0.17',
-   }),
-    ('garage', {
-        'mac': 'b8:27:eb:81:17:92',
-        'ssh_hostname': '10.2.0.61',
-        'wireguard_address': '10.5.0.14',
-    }),
+#   ('frontbed', {
+#       'mac': 'b8:27:eb:e9:d3:44',
+#       'ssh_hostname': '10.2.0.27',
+#       'wireguard_address': '10.5.0.17',
+#   }),
+#    ('garage', {
+#        'mac': 'b8:27:eb:81:17:92',
+#        'ssh_hostname': '10.2.0.61',
+#        'wireguard_address': '10.5.0.14',
+#    }),
 ]
 
 remote = [
@@ -45,8 +47,8 @@
         'ssh_hostname': '162.243.138.136',
         'wireguard_address': '10.5.0.2',
     }),
-    ('plus', {
-        'ssh_hostname': '10.2.0.40',
-        'wireguard_address': '10.5.0.110',
-    }),
+#    ('plus', {
+#        'ssh_hostname': '10.2.0.40',
+#        'wireguard_address': '10.5.0.110',
+#    }),
 ]
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/multikube.py	Sat Aug 20 14:12:05 2022 -0700
@@ -0,0 +1,114 @@
+# leave kube.py running single-host and try again
+import os
+
+from pyinfra import host
+from pyinfra.facts.files import FindInFile
+from pyinfra.facts.server import Arch, LinuxDistribution
+from pyinfra.operations import files, server, systemd
+
+is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
+
+from multikube_config import server_node, master_ip,nodes, admin_from, k3s_version, skaffold_version
+
+def download_k3s():
+    tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
+    files.download(
+        src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
+        dest='/usr/local/bin/k3s',
+        user='root',
+        group='root',
+        mode='755',
+        cache_time=43000,
+        #force=True,  # to get a new version
+    )
+
+
+def install_skaffold():
+    files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
+                   dest='/usr/local/bin/skaffold',
+                   user='root',
+                   group='root',
+                   mode='755',
+                   cache_time=1000)
+    # one time; writes to $HOME
+    #skaffold config set --global insecure-registries bang5:5000
+
+
+def pi_cgroup_setup():
+    old_cmdline = host.get_fact(FindInFile, path='/boot/cmdline.txt', pattern=r'.*')[0]
+    if 'cgroup' not in old_cmdline:
+        cmdline = old_cmdline + ' cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
+        files.line(path='/boot/cmdline.txt', line='.*', replace=cmdline)
+        # pi needs reboot now
+
+
+def host_prep():
+    server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
+    server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
+    server.sysctl(key='net.ipv6.conf.all.disable_ipv6' , value='1',persist=True)
+    server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
+    server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)
+
+    # https://sysctl-explorer.net/net/ipv4/rp_filter/
+    #none, strict, loose = 0, 1, 2
+    #server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)
+
+    if is_pi:
+        pi_cgroup_setup()
+
+def service_name():
+    return 'k3s.service' if host.name == server_node else 'k3s-node.service'
+
+def config_and_run_service():
+    download_k3s()
+    role = 'server' if host.name == server_node else 'agent'
+    which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'
+
+    if host.name == server_node:
+        token = "unused"
+    else:
+        token = open('/tmp/k3s-token', 'rt').read().strip()
+    files.template(
+        src=f'templates/kube/{which_conf}',
+        dest='/etc/k3s_config.yaml',
+        master_ip=master_ip,
+        token=token,
+        wg_ip=host.host_data['mk_addr'],#wireguard_address'],
+    )
+
+    files.template(
+        src='templates/kube/k3s.service.j2',
+        dest=f'/etc/systemd/system/{service_name()}',
+        role=role,
+    )
+    systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True)
+
+    if host.name == server_node:
+       files.get(src='/var/lib/rancher/k3s/server/node-token', dest='/tmp/k3s-token')
+       files.get(src='/etc/rancher/k3s/k3s.yaml', dest='/tmp/k3s-yaml')
+
+if host.name in nodes + [server_node]:
+    host_prep()
+    files.directory(path='/etc/rancher/k3s')
+
+    config_and_run_service()
+
+    # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
+    # user confusions: https://github.com/rancher/k3s/issues/1802
+    files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
+    # for the possible registries update:
+    systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True)
+
+if host.name in admin_from:
+    files.directory(path='/etc/rancher/k3s')
+    install_skaffold()
+    files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
+    files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
+    # .zshrc has: export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+
+    # assumes our pyinfra process is running on server_node
+    if host.name != server_node:
+        files.put(src='/tmp/k3s-yaml', dest='/etc/rancher/k3s/k3s.yaml')
+
+    files.file(path='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640')
+    server.shell(f"kubectl config set-cluster default --server=https://{master_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/multikube_config.py	Sat Aug 20 14:12:05 2022 -0700
@@ -0,0 +1,10 @@
+
+server_node = 'slash'
+master_ip = "10.2.0.84"
+nodes = ['dash']
+admin_from = ['slash', 'dash']
+
+k3s_version = 'v1.24.3+k3s1'
+
+# https://github.com/GoogleContainerTools/skaffold/releases
+skaffold_version = 'v1.39.1'
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/multikube_wipe.py	Sat Aug 20 14:12:05 2022 -0700
@@ -0,0 +1,12 @@
+from pyinfra import host
+from pyinfra.operations import files, server, systemd
+
+from multikube_config import server_node, nodes
+
+def k3s_wipe():
+    files.put(dest='/usr/local/bin/k3s-killall.sh', src='files/kube/k3s-killall.sh', mode='a+rx')
+    files.put(dest='/usr/local/bin/k3s-uninstall.sh', src='files/kube/k3s-uninstall.sh', mode='a+rx')
+    server.shell(['k3s-uninstall.sh'])
+
+if host.name in nodes + [server_node]:
+    k3s_wipe()
--- a/tasks.py	Sat Aug 20 14:13:02 2022 -0700
+++ b/tasks.py	Sat Aug 20 14:12:05 2022 -0700
@@ -82,6 +82,13 @@
 def pipe(ctx):
     ctx.run(cmd + 'inventory.py pipe.py --limit pipe', pty=True)
 
+@task
+def multikube(ctx):
+    from multikube_config import server_node, nodes
+    ctx.run(cmd + 'inventory.py multikube_wipe.py', pty=True)
+    ctx.run(cmd + f'inventory.py multikube.py --limit {server_node}', pty=True)
+    ctx.run(cmd + f'inventory.py multikube.py --limit {",".join(nodes)}', pty=True)
+
 
 @task
 def all(ctx):