view tasks.py @ 12:b6720e379d5b

config updates
author drewp@bigasterisk.com
date Tue, 14 Mar 2023 20:04:06 -0700
parents 2023a6ce7bc0
children 2c37fab420da
line wrap: on
line source

import json
import sys
import tempfile
import time
import urllib.request
from pathlib import Path
from typing import Dict

from invoke import task

sys.path.append('/usr/lib/python3/dist-packages/')
import yaml

_tfs = []


def saveTmp(text):
    tf = tempfile.NamedTemporaryFile(mode='wt')
    _tfs.append(tf)
    tf.write(text)
    tf.flush()
    return Path(tf.name)


def writeConfigmap(ctx, files: Dict[str, Path]):
    arg = ','.join(f'{k}={v}' for k, v in files.items())
    ctx.run(
        f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -'
    )


def reload(ctx, svc):
    host = ctx.run(f'khost {svc}').stdout
    path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc]
    reload_url = f'http://{host}{path}-/reload'
    print(f'reload with POST {reload_url}')
    for workaround in [1]:
        print(' -> status',
              urllib.request.urlopen(reload_url, data=b'unused').status)
        time.sleep(0)


def httpsCertProber():
    domains = []
    for line in open(
            '/my/doc/ssl/letsencrypt/run.py'):  # moved to cert-manager
        if line.startswith('update_certs('):
            domains.append(line.split("'")[1])
    relabel = {
        'relabel_configs': [{
            'source_labels': ['__address__'],
            'target_label': '__param_target'
        }, {
            'source_labels': ['__param_target'],
            'target_label': 'instance'
        }, {
            'target_label': '__address__',
            'replacement': 'prober'
        }]
    }
    return yaml.dump(  # Note that an included file must skip the scrape_configs toplevel key and just include the list.
        [{
            'job_name': 'prober',
            'scrape_interval': '24h',
            'metrics_path': '/probe',
            'params': {
                'module': ['https']
            },
            'static_configs': [{
                'targets': domains
            }],
        } | relabel])


def hostsExpectedOnline(ctx):
    return ctx.run(
        'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout


def expectedK8sNodes(ctx):
    getNode = json.loads(ctx.run("kubectl get node -o json").stdout)
    hosts = [item['metadata']['name'] for item in getNode['items']]
    return yaml.dump({
        'groups': [{
            'name':
            'k8s_expected_nodes',
            'rules': [{
                'alert':
                'kube_node_log_size_report_' + h,
                'expr':
                'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})'
                % h,
                'for':
                '1h',
                'annotations': {
                    'summary': f"no recent k8s log size report from host {h}"
                }
            } for h in hosts]
        }]
    })


@task
def sync_config(ctx):
    config = Path('config')
    for workaround in [1]:
        writeConfigmap(
            ctx, {
#                'scrape_ssl.yaml': saveTmp(httpsCertProber()),
                'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)),
                'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)),
            })
        reload(ctx, 'victoriametrics')

        # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem
        # reload(ctx, 'vmalert')
        ctx.run('kubectl rollout restart deploy/vmalert')

@task
def build_config(ctx):
    with open('rules/build/expected_hosts.yaml', 'w') as out:
        out.write(hostsExpectedOnline(ctx))