view multikube.py @ 332:d4893670f888 default tip

WIP: use watchdog reboot timer on pi
author drewp@bigasterisk.com
date Thu, 27 Feb 2025 11:09:29 -0800
parents 5b88b38f2471
children
line wrap: on
line source

# leave kube.py running single-host and try again
from pyinfra.context import host
from pyinfra.facts.server import Arch
from pyinfra.operations import files, server, systemd

from multikube_config import (
    admin_from,
    k3s_version,
    nodes,
    server_ip,
    server_node,
    skaffold_version,
)


def install_k3s():
    tail = 'k3s' if host.get_fact(Arch) == 'x86_64' else 'k3s-armhf'
    files.download(
        src=f'https://github.com/rancher/k3s/releases/download/{k3s_version}/{tail}',
        dest='/usr/local/bin/k3s',
        user='root',
        group='root',
        mode='755',
        cache_time=43000,
        #force=True,  # to get a new version
    )


def install_k3sup():
    files.download(src='https://github.com/alexellis/k3sup/releases/download/0.12.0/k3sup',
                   dest='/usr/local/bin/k3sup',
                   mode='755')
    # then do like:
    #  root@slash:/home/drewp# ./k3sup install --ip 10.2.0.84 --k3s-extra-args '--no-deploy traefik' --ssh-key /root/.ssh/id_ecdsa
    #  root@slash:/home/drewp# ./k3sup join --ip 10.2.0.23 --server-ip 10.2.0.84  --ssh-key /root/.ssh/id_ecdsa
    # but it doesn't seem to make networking go any better


def install_skaffold():
    files.download(src=f'https://storage.googleapis.com/skaffold/releases/{skaffold_version}/skaffold-linux-amd64',
                   dest='/usr/local/bin/skaffold',
                   user='root',
                   group='root',
                   mode='755',
                   cache_time=1000)
    # one time; writes to $HOME
    #skaffold config set --global insecure-registries bang5:5000


def host_prep():
    server.sysctl(key='net.ipv4.ip_forward', value="1", persist=True)
    server.sysctl(key='net.ipv6.conf.all.forwarding', value="1", persist=True)
    server.sysctl(key='net.ipv6.conf.all.disable_ipv6', value='1', persist=True)
    server.sysctl(key='fs.inotify.max_user_instances', value='8192', persist=True)
    server.sysctl(key='fs.inotify.max_user_watches', value='524288', persist=True)

    # https://sysctl-explorer.net/net/ipv4/rp_filter/
    #none, strict, loose = 0, 1, 2
    #server.sysctl(key='net.ipv4.conf.default.rp_filter', value=loose, persist=True)


def service_name():
    return 'k3s.service' if host.name == server_node else 'k3s-node.service'


def config_and_run_service():
    role = 'server' if host.name == server_node else 'agent'
    which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'

    if host.name == server_node:
        token = "unused"
    else:
        token = open('/tmp/k3s-token', 'rt').read().strip()
    files.template(
        src=f'templates/kube/{which_conf}',
        dest='/etc/k3s_config.yaml',
        server_ip=server_ip,
        token=token,
        wg_ip=host.host_data['mk_addr'],  #wireguard_address'],
    )

    files.template(
        src='templates/kube/k3s.service.j2',
        dest=f'/etc/systemd/system/{service_name()}',
        role=role,
    )
    systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True)

    if host.name == server_node:
        files.get(src='/var/lib/rancher/k3s/server/node-token', dest='/tmp/k3s-token')
        files.get(src='/etc/rancher/k3s/k3s.yaml', dest='/tmp/k3s-yaml')


if host.name in nodes + [server_node]:
    host_prep()
    files.directory(path='/etc/rancher/k3s')
    install_k3s()
    config_and_run_service()

    # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
    # user confusions: https://github.com/rancher/k3s/issues/1802
    files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
    # for the possible registries update:
    systemd.service(service=service_name(), daemon_reload=True, enabled=True, restarted=True)

if host.name in admin_from:
    files.directory(path='/etc/rancher/k3s')
    install_skaffold()
    files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
    files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
    # .zshrc has: export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

    if host.name != server_node:
        files.put(src='/tmp/k3s-yaml', dest='/etc/rancher/k3s/k3s.yaml')

    files.file(path='/etc/rancher/k3s/k3s.yaml', user='root', group='drewp', mode='640')
    server.shell(
        commands=f"kubectl config set-cluster default --server=https://{server_ip}:6443 --kubeconfig=/etc/rancher/k3s/k3s.yaml")