Mercurial > code > home > repos > victoriametrics
view tasks.py @ 12:b6720e379d5b
config updates
author | drewp@bigasterisk.com |
---|---|
date | Tue, 14 Mar 2023 20:04:06 -0700 |
parents | 2023a6ce7bc0 |
children | 2c37fab420da |
line wrap: on
line source
import json import sys import tempfile import time import urllib.request from pathlib import Path from typing import Dict from invoke import task sys.path.append('/usr/lib/python3/dist-packages/') import yaml _tfs = [] def saveTmp(text): tf = tempfile.NamedTemporaryFile(mode='wt') _tfs.append(tf) tf.write(text) tf.flush() return Path(tf.name) def writeConfigmap(ctx, files: Dict[str, Path]): arg = ','.join(f'{k}={v}' for k, v in files.items()) ctx.run( f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -' ) def reload(ctx, svc): host = ctx.run(f'khost {svc}').stdout path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc] reload_url = f'http://{host}{path}-/reload' print(f'reload with POST {reload_url}') for workaround in [1]: print(' -> status', urllib.request.urlopen(reload_url, data=b'unused').status) time.sleep(0) def httpsCertProber(): domains = [] for line in open( '/my/doc/ssl/letsencrypt/run.py'): # moved to cert-manager if line.startswith('update_certs('): domains.append(line.split("'")[1]) relabel = { 'relabel_configs': [{ 'source_labels': ['__address__'], 'target_label': '__param_target' }, { 'source_labels': ['__param_target'], 'target_label': 'instance' }, { 'target_label': '__address__', 'replacement': 'prober' }] } return yaml.dump( # Note that an included file must skip the scrape_configs toplevel key and just include the list. [{ 'job_name': 'prober', 'scrape_interval': '24h', 'metrics_path': '/probe', 'params': { 'module': ['https'] }, 'static_configs': [{ 'targets': domains }], } | relabel]) def hostsExpectedOnline(ctx): return ctx.run( 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout def expectedK8sNodes(ctx): getNode = json.loads(ctx.run("kubectl get node -o json").stdout) hosts = [item['metadata']['name'] for item in getNode['items']] return yaml.dump({ 'groups': [{ 'name': 'k8s_expected_nodes', 'rules': [{ 'alert': 'kube_node_log_size_report_' + h, 'expr': 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' % h, 'for': '1h', 'annotations': { 'summary': f"no recent k8s log size report from host {h}" } } for h in hosts] }] }) @task def sync_config(ctx): config = Path('config') for workaround in [1]: writeConfigmap( ctx, { # 'scrape_ssl.yaml': saveTmp(httpsCertProber()), 'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)), 'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)), }) reload(ctx, 'victoriametrics') # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem # reload(ctx, 'vmalert') ctx.run('kubectl rollout restart deploy/vmalert') @task def build_config(ctx): with open('rules/build/expected_hosts.yaml', 'w') as out: out.write(hostsExpectedOnline(ctx))