Mercurial > code > home > repos > infra

changeset 103:8b8ef9d8f0fd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
dead code and templates, reformat, maybe a little refactor
author drewp@bigasterisk.com
date Fri, 15 Jul 2022 15:24:44 -0700
parents 9ec487a381e2
children d56b72c72d9b
files dns.py kube.py net.py packages.py pipe.py sync.py tasks.py templates/kube/config-agent.yaml.j2 templates/kube/config-server.yaml.j2 templates/kube/config.yaml.j2 templates/kube/node-config.yaml.j2 templates/net/bang_isp.network.j2 templates/resolv.conf.j2
diffstat 13 files changed, 107 insertions(+), 162 deletions(-) [+]
line wrap: on
line diff
--- a/dns.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/dns.py	Fri Jul 15 15:24:44 2022 -0700
@@ -2,30 +2,15 @@
 from pyinfra.operations import apt, files, systemd
 
 
-def resolv_conf_use_systemd_networkd():
-    files.link(path='/etc/resolv.conf', target='/run/systemd/resolve/resolv.conf', force=True)
-
-
-def resolv_conf_static_file():
-    files.file(path='/etc/resolv.conf', present=False, force=True)
-    files.template(
-        src='templates/resolv.conf.j2',
-        dest='/etc/resolv.conf',
-        # review this- it's probably a bad dep on bang. maybe both 10.5.0.1 and a public ns would be ok
-        ns='10.5.0.1' if host.name in ['prime', 'plus'] else '10.2.0.3',
-        force=True)
-
-
 def dnsmasq_instance(net_name, house_iface, dhcp_range, router):
     files.directory(path=f'/opt/dnsmasq/{net_name}')
-    files.template(
-        src='templates/dnsmasq/dnsmasq.conf.j2',
-        dest=f'/opt/dnsmasq/{net_name}/dnsmasq.conf',
-        net=net_name,
-        house_iface=house_iface,
-        dhcp_range=dhcp_range,
-        router=router,
-        dhcp_enabled=net_name == '10.2' and host.name == 'pipe')
+    files.template(src='templates/dnsmasq/dnsmasq.conf.j2',
+                   dest=f'/opt/dnsmasq/{net_name}/dnsmasq.conf',
+                   net=net_name,
+                   house_iface=house_iface,
+                   dhcp_range=dhcp_range,
+                   router=router,
+                   dhcp_enabled=net_name == '10.2' and host.name == 'pipe')
     files.template(src='templates/dnsmasq/hosts.j2', dest=f'/opt/dnsmasq/{net_name}/hosts', net=net_name)
     files.template(src='/dev/null', dest=f'/opt/dnsmasq/{net_name}/dhcp_hosts', net=net_name)
 
@@ -37,7 +22,7 @@
 
 
 files.template(src='templates/hosts.j2', dest='/etc/hosts')
-resolv_conf_use_systemd_networkd()
+files.link(path='/etc/resolv.conf', target='/run/systemd/resolve/resolv.conf', force=True)
 files.template(src='templates/resolved.conf.j2', dest='/etc/systemd/resolved.conf')
 systemd.service(service='systemd-resolved.service', running=True, restarted=True)
 
--- a/kube.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/kube.py	Fri Jul 15 15:24:44 2022 -0700
@@ -1,5 +1,5 @@
 import os
-import tempfile
+
 from pyinfra import host
 from pyinfra.facts.files import FindInFile
 from pyinfra.facts.server import Arch, LinuxDistribution
@@ -75,8 +75,8 @@
 def config_and_run_service():
     download_k3s()
     service_name = 'k3s.service' if host.name == server_node else 'k3s-node.service'
-    which_conf = 'config.yaml.j2' if host.name == server_node else 'node-config.yaml.j2'
     role = 'server' if host.name == server_node else 'agent'
+    which_conf = 'config-server.yaml.j2' if host.name == server_node else 'config-agent.yaml.j2'
 
     # /var/lib/rancher/k3s/server/node-token is the source of the string in secrets/k3s_token,
     # so this presumes a previous run
@@ -94,9 +94,6 @@
         token=token,
         wg_ip=host.host_data['wireguard_address'],
     )
-    # files.put(
-    #     src='templates/kube/flannel.link',  #
-    #     dest='/etc/systemd/network/10-flannel.link')  # then reboot
     files.template(
         src='templates/kube/k3s.service.j2',
         dest=f'/etc/systemd/system/{service_name}',
@@ -105,48 +102,23 @@
     systemd.service(service=service_name, daemon_reload=True, enabled=True, restarted=True)
 
 
-# See https://github.com/rancher/k3s/issues/1802 and https://rancher.com/docs/k3s/latest/en/installation/private-registry/
-files.directory(path='/etc/rancher/k3s')
-
 if host.name in nodes + [server_node]:
     host_prep()
+    files.directory(path='/etc/rancher/k3s')
 
-    # not until registry is up, right?
+    # docs: https://rancher.com/docs/k3s/latest/en/installation/private-registry/
+    # user confusions: https://github.com/rancher/k3s/issues/1802
     files.template(src='templates/kube/registries.yaml.j2', dest='/etc/rancher/k3s/registries.yaml')
     config_and_run_service()
 
-# if host.name == server_node:
-#     files.put(
-#         src="templates/kube/coredns.yaml",
-#         dest="/var/lib/rancher/k3s/server/manifests/coredns.yaml",
-#         mode="600",
-#     )
-    # files.put(
-    #     src="templates/kube/coredns-map.yaml",
-    #     dest="/var/lib/rancher/k3s/server/manifests/coredns-map.yaml",
-    #     mode="600",
-    # )
-    # tmp = tempfile.NamedTemporaryFile(suffix='.yaml')
-    # files.template(
-    #     src='templates/kube/Corefile.yaml.j2',
-    #     dest=tmp.name,
-    # )
-    # server.shell(commands=[
-    #     'kubectl replace configmap '
-    #     # '-n kube-system '
-    #     # 'coredns '
-    #     f'--filename={tmp.name} '
-    #     '-o yaml '
-    #     # '--dry-run=client | kubectl apply -',
-    # ])
-
 if host.name in admin_from:
+    files.directory(path='/etc/rancher/k3s')
     install_skaffold()
     files.link(path='/usr/local/bin/kubectl', target='/usr/local/bin/k3s')
     files.directory(path='/home/drewp/.kube', user='drewp', group='drewp')
     files.line(path="/home/drewp/.zshrc", line="KUBECONFIG", replace='export KUBECONFIG=/etc/rancher/k3s/k3s.yaml')
 
-    # assumes pyinfra is running on server_node
+    # assumes our pyinfra process is running on server_node
     files.put(
         src='/etc/rancher/k3s/k3s.yaml',
         dest='/etc/rancher/k3s/k3s.yaml',  #
--- a/net.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/net.py	Fri Jul 15 15:24:44 2022 -0700
@@ -1,7 +1,8 @@
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, server, ssh, systemd
 
 is_wifi = host.name in ['frontdoor', 'living', 'plus']
+is_wifi_pi = host.name in ['frontdoor']
 
 
 def cleanup():
@@ -33,8 +34,13 @@
     #   https://cloud.digitalocean.com/networking/firewalls/f68899ae-1aac-4469-b379-59ce2bbc988f/droplets?i=7c5072
     apt.packages(packages=['ufw'], present=False)
 
+
 server.sysctl(key='net.ipv6.conf.all.disable_ipv6', value=1, persist=True)
 
+if is_wifi_pi:
+    files.put(dest="/etc/network/interfaces.d/wlan0", src="files/pi_wlan0_powersave")
+    ssh.command(host.name, "iw wlan0 set power_save off")
+    
 files.directory('/etc/systemd/network')
 if host.name == 'prime':
     cleanup()
@@ -56,7 +62,7 @@
 elif host.name == 'plus':
     pass
 
-elif host.name == 'pipe':   
+elif host.name == 'pipe':
     cleanup()
 
     files.template(src="templates/net/pipe_10.2.network.j2", dest="/etc/systemd/network/99-10.2.network")
@@ -65,7 +71,6 @@
     files.template(src="templates/net/house_net.service.j2", dest="/etc/systemd/system/house_net.service", out_interface='eth0')
     systemd.service(service='house_net.service', daemon_reload=True, enabled=True, running=True, restarted=True)
 
-
 else:
     cleanup()
 
@@ -78,4 +83,4 @@
         create_remote_dir=True,
     )
 
-systemd.service(service='systemd-networkd.service', enabled=True, running=True, restarted=True)
\ No newline at end of file
+systemd.service(service='systemd-networkd.service', enabled=True, running=True, restarted=True)
--- a/packages.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/packages.py	Fri Jul 15 15:24:44 2022 -0700
@@ -1,18 +1,39 @@
 from pyinfra import host
 from pyinfra.facts.server import LinuxDistribution
-from pyinfra.operations import apt, files, ssh, server
+from pyinfra.operations import apt, files, server
 
 is_pi = host.get_fact(LinuxDistribution)['name'] in ['Debian', 'Raspbian GNU/Linux']
-is_wifi_pi = host.name in ['frontdoor', 'living']
+
 
-if not is_pi:
-    apt.key(keyserver='keyserver.ubuntu.com', keyid='8B48AD6246925553')
+def kitty():
+    vers = '0.25.2'  # see https://github.com/kovidgoyal/kitty/releases
+    home = '/home/drewp'
+    local = f"{home}/.local/kitty"
+    dl = f'/tmp/kitty-{vers}-x86_64.txz'
+    files.download(src=f"https://github.com/kovidgoyal/kitty/releases/download/v{vers}/kitty-{vers}-x86_64.txz", dest=dl)
+    files.directory(local)
+    server.shell([
+        f"mkdir -p {local}",  # https://github.com/Fizzadar/pyinfra/issues/777
+        f"aunpack --extract-to={local} {dl}",
+    ])
+    files.link(target="{local}/bin/kitty", path="{home}/bin/kitty")
+
 
-if host.name == 'pipe':
-    apt.packages(packages=['mandb', 'apt-listchanges'], present=False)
-    files.template(src='templates/odroid_sources.list.j2', dest='/etc/apt/sources.list', rel='focal')
+def pnpm():
+    server.shell([
+        # https://github.com/pnpm/pnpm/releases
+        # but also https://pnpm.io/installation#compatibility
+        "npm install -g pnpm@6.32.22",
+    ])
+
 
-elif is_pi:
+def proper_locate():
+    apt.packages(packages='mlocate', present=False)
+    if not is_pi and host.name not in ['prime', 'pipe']:
+        apt.packages(packages='plocate')
+
+
+def pi_sources():
     apt.packages(packages=['mandb', 'apt-listchanges'], present=False)
     files.template(src='templates/pi_sources.list.j2', dest='/etc/apt/sources.list', rel='bullseye')
     # 'apt upgrade'?
@@ -28,40 +49,29 @@
 
     files.file(path='/etc/apt/sources.list.d/raspi.list', present=False)
 
-    if is_wifi_pi:
-        files.put(dest="/etc/network/interfaces.d/wlan0", src="files/pi_wlan0_powersave")
-        ssh.command(host.name, "iw wlan0 set power_save off")
-
     files.template(src='templates/boot_config.txt.j2', dest='/boot/config.txt')
 
-if not is_pi and host.name != 'prime':
-    apt.key(src='https://dl.google.com/linux/linux_signing_key.pub')
-    apt.repo(src='deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main')
 
-    apt.key(src='https://packages.microsoft.com/keys/microsoft.asc')
-    apt.repo(src="deb [arch=amd64,arm64,armhf] http://packages.microsoft.com/repos/code stable main")
-
-    apt.key(keyserver='keyserver.ubuntu.com', keyid='F24AEA9FB05498B7')
-    apt.repo(src="deb [arch=amd64,i386] https://repo.steampowered.com/steam/ stable steam")
+if not is_pi:
+    apt.key(keyserver='keyserver.ubuntu.com', keyid='8B48AD6246925553')
 
-apt.packages(packages=[
-    'build-essential',
-    # 'i2c-tools',
-    'rsync',
-    'dstat',
-    'ifstat',
-    'mosquitto-clients',
-    'ncdu',
-    "udns-utils",
-    "atool",
-    "wireguard-tools",
-    'iproute2', # needed for wireguard
-    'kitty-terminfo',
-    'zsh',
-    'xosview',
-])
+if host.name == 'pipe':
+    apt.packages(packages=['mandb', 'apt-listchanges'], present=False)
+    files.template(src='templates/odroid_sources.list.j2', dest='/etc/apt/sources.list', rel='focal')
+elif is_pi:
+    pi_sources()
 
 if not is_pi:
+    if host.name != 'prime':
+        apt.key(src='https://dl.google.com/linux/linux_signing_key.pub')
+        apt.repo(src='deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main')
+
+        apt.key(src='https://packages.microsoft.com/keys/microsoft.asc')
+        apt.repo(src="deb [arch=amd64,arm64,armhf] http://packages.microsoft.com/repos/code stable main")
+
+        apt.key(keyserver='keyserver.ubuntu.com', keyid='F24AEA9FB05498B7')
+        apt.repo(src="deb [arch=amd64,i386] https://repo.steampowered.com/steam/ stable steam")
+
     apt.packages(packages=[
         'debian-goodies',
         'ethtool',
@@ -91,39 +101,39 @@
             'reptyr',
         ])
 
-    vers = '0.25.2' # see https://github.com/kovidgoyal/kitty/releases
-    home = '/home/drewp'
-    local = f"{home}/.local/kitty"
-    dl = f'/tmp/kitty-{vers}-x86_64.txz'
-    files.download(src=f"https://github.com/kovidgoyal/kitty/releases/download/v{vers}/kitty-{vers}-x86_64.txz",
-                   dest=dl)
-    files.directory(local)
-    server.shell([
-        f"mkdir -p {local}",  # https://github.com/Fizzadar/pyinfra/issues/777
-        f"aunpack --extract-to={local} {dl}",
-    ])
-    files.link(target="{local}/bin/kitty", path="{home}/bin/kitty")
+    kitty()
+    pnpm()
 
-    server.shell([
-        # https://github.com/pnpm/pnpm/releases
-        # but also https://pnpm.io/installation#compatibility
-        "npm install -g pnpm@6.32.22",
-        ])
+apt.packages(packages=[
+    'build-essential',
+    'dstat',
+    'ifstat',
+    'iproute2',  # needed for wireguard
+    'kitty-terminfo',
+    'mosquitto-clients',
+    'ncdu',
+    'rsync',
+    'xosview',
+    'zsh',
+    "atool",
+    "udns-utils",
+    "wireguard-tools",
+])
 
-apt.packages(packages='mlocate', present=False)
-if not is_pi and host.name not in ['prime', 'pipe']:
-    apt.packages(packages='plocate')
+proper_locate()
 
 if host.name == "bang":
     apt.packages(packages=[
         'dnsmasq',
         'iptables',
+        'ntpdate',
         'openntpd',
-        'ntpdate',
+        'zfs-auto-snapshot',
+        'zfs-zed',
         'zfsutils-linux',
-        'zfs-zed',
-        'zfs-auto-snapshot',
     ])
 
 if host.name == 'plus':
-    apt.packages(packages=['network-manager'])
+    apt.packages(packages=[
+        'network-manager',
+    ])
--- a/pipe.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/pipe.py	Fri Jul 15 15:24:44 2022 -0700
@@ -1,15 +1,9 @@
-import os
-import tempfile
-
-from pyinfra import host
-from pyinfra.facts.files import FindInFile
-from pyinfra.facts.server import Arch, LinuxDistribution
-from pyinfra.operations import apt, files, server, systemd, git
+from pyinfra.operations import apt, files, git, server
 
 
 def ntop():
     files.directory('/opt/ntop')
-    
+
     apt.packages(packages=[
         "build-essential", "git", "bison", "flex", "libxml2-dev", "libpcap-dev", "libtool", "libtool-bin", "rrdtool",
         "librrd-dev", "autoconf", "pkg-config", "automake", "autogen", "redis-server", "wget", "libsqlite3-dev", "libhiredis-dev",
--- a/sync.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/sync.py	Fri Jul 15 15:24:44 2022 -0700
@@ -1,5 +1,4 @@
 from pyinfra import host
-from pyinfra.facts.server import LinuxDistribution
 from pyinfra.operations import apt, systemd
 
 # primary instance is in k8s (/my/serv/filesync/syncthing); the rest are run with systemd.
--- a/tasks.py	Fri Jul 15 14:51:12 2022 -0700
+++ b/tasks.py	Fri Jul 15 15:24:44 2022 -0700
@@ -42,6 +42,7 @@
 def dns_check(ctx):
     ctx.run(cmd + 'inventory.py dns_check.py -v', pty=True)
 
+
 @task
 def dns_k8s_check(ctx):
     ctx.run('env/bin/python dns_k8s_check.py', pty=True)
@@ -56,6 +57,7 @@
 def kube(ctx):
     ctx.run(cmd + 'inventory.py kube.py ', pty=True)
 
+
 @task
 def kube_bang(ctx):
     ctx.run(cmd + 'inventory.py kube.py --limit bang', pty=True)
@@ -70,6 +72,7 @@
 def mail(ctx):
     ctx.run(cmd + 'inventory.py mail.py ', pty=True)
 
+
 @task
 def pipe(ctx):
     ctx.run(cmd + 'inventory.py pipe.py --limit pipe', pty=True)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/templates/kube/config-agent.yaml.j2	Fri Jul 15 15:24:44 2022 -0700
@@ -0,0 +1,3 @@
+node-ip: {{ wg_ip }}
+token: {{ token }}
+server: https://{{ master_ip }}:6443 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/templates/kube/config-server.yaml.j2	Fri Jul 15 15:24:44 2022 -0700
@@ -0,0 +1,5 @@
+write-kubeconfig-mode: '640'
+node-ip: {{ wg_ip }}
+disable:
+  - traefik
+flannel-backend: host-gw
--- a/templates/kube/config.yaml.j2	Fri Jul 15 14:51:12 2022 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,5 +0,0 @@
-write-kubeconfig-mode: '640'
-node-ip: {{ wg_ip }}
-disable:
-  - traefik
-flannel-backend: host-gw
--- a/templates/kube/node-config.yaml.j2	Fri Jul 15 14:51:12 2022 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,3 +0,0 @@
-node-ip: {{ wg_ip }}
-token: {{ token }}
-server: https://{{ master_ip }}:6443 
--- a/templates/net/bang_isp.network.j2	Fri Jul 15 14:51:12 2022 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,11 +0,0 @@
-# written by pyinfra
-
-[Match]
-MACAddress=64:ee:b7:14:81:9e
-
-[Network]
-DHCP=no
-Address=192.168.42.2/24
-Gateway=192.168.42.1
-DNS=10.2.0.1
-DNSDefaultRoute=true
--- a/templates/resolv.conf.j2	Fri Jul 15 14:51:12 2022 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,12 +0,0 @@
-# written by pyinfra
-
-# see 'resolvectl status' for more
-
-nameserver {{ ns }}
-{% if host.name == 'prime' %}
-# don't rely on home dns just to receive email
-nameserver 8.8.4.4
-{% endif %}
-
-options edns0
-search bigasterisk.com