Mercurial > code > home > repos > victoriametrics

comparison tasks.py @ 12:b6720e379d5b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
config updates
author drewp@bigasterisk.com
date Tue, 14 Mar 2023 20:04:06 -0700
parents 2023a6ce7bc0
children 2c37fab420da
comparison
equal deleted inserted replaced
11:2eab3e6b89f2 12:b6720e379d5b
6 from pathlib import Path 6 from pathlib import Path
7 from typing import Dict 7 from typing import Dict
8 8
9 from invoke import task 9 from invoke import task
10 10
11 sys.path.append('/usr/lib/python3/dist-packages/')
12 import yaml
11 13
12 def updateConfigmapCmd(path): 14 _tfs = []
13 return f'kubectl create configmap victoriametrics-config --from-file {path}=config/{path} -o yaml --dry-run=client | kubectl apply -f -' 15
16
17 def saveTmp(text):
18 tf = tempfile.NamedTemporaryFile(mode='wt')
19 _tfs.append(tf)
20 tf.write(text)
21 tf.flush()
22 return Path(tf.name)
23
24
25 def writeConfigmap(ctx, files: Dict[str, Path]):
26 arg = ','.join(f'{k}={v}' for k, v in files.items())
27 ctx.run(
28 f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -'
29 )
14 30
15 31
16 def reload(ctx, svc): 32 def reload(ctx, svc):
17 host = ctx.run(f'khost {svc}').stdout 33 host = ctx.run(f'khost {svc}').stdout
18 path = {'victoriametrics': '/m', 'vmalert': '/vmalert'}[svc] 34 path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc]
19 print(' -> status', 35 reload_url = f'http://{host}{path}-/reload'
20 urllib.request.urlopen(f'http://{host}{path}/-/reload').status) 36 print(f'reload with POST {reload_url}')
37 for workaround in [1]:
38 print(' -> status',
39 urllib.request.urlopen(reload_url, data=b'unused').status)
40 time.sleep(0)
41
42
43 def httpsCertProber():
44 domains = []
45 for line in open(
46 '/my/doc/ssl/letsencrypt/run.py'): # moved to cert-manager
47 if line.startswith('update_certs('):
48 domains.append(line.split("'")[1])
49 relabel = {
50 'relabel_configs': [{
51 'source_labels': ['__address__'],
52 'target_label': '__param_target'
53 }, {
54 'source_labels': ['__param_target'],
55 'target_label': 'instance'
56 }, {
57 'target_label': '__address__',
58 'replacement': 'prober'
59 }]
60 }
61 return yaml.dump( # Note that an included file must skip the scrape_configs toplevel key and just include the list.
62 [{
63 'job_name': 'prober',
64 'scrape_interval': '24h',
65 'metrics_path': '/probe',
66 'params': {
67 'module': ['https']
68 },
69 'static_configs': [{
70 'targets': domains
71 }],
72 } | relabel])
73
74
21 def hostsExpectedOnline(ctx): 75 def hostsExpectedOnline(ctx):
22 return ctx.run( 76 return ctx.run(
23 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout 77 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout
24 78
25 79
80 def expectedK8sNodes(ctx):
81 getNode = json.loads(ctx.run("kubectl get node -o json").stdout)
82 hosts = [item['metadata']['name'] for item in getNode['items']]
83 return yaml.dump({
84 'groups': [{
85 'name':
86 'k8s_expected_nodes',
87 'rules': [{
88 'alert':
89 'kube_node_log_size_report_' + h,
90 'expr':
91 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})'
92 % h,
93 'for':
94 '1h',
95 'annotations': {
96 'summary': f"no recent k8s log size report from host {h}"
97 }
98 } for h in hosts]
99 }]
100 })
26 101
27 102
28 @task 103 @task
29 def sync_config(ctx): 104 def sync_config(ctx):
30 ctx.run(updateConfigmapCmd("scrape_config.yaml")) 105 config = Path('config')
31 ctx.run(updateConfigmapCmd("scrape_ssl.yaml")) 106 for workaround in [1]:
32 reload(ctx, 'victoriametrics') 107 writeConfigmap(
108 ctx, {
109 # 'scrape_ssl.yaml': saveTmp(httpsCertProber()),
110 'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)),
111 'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)),
112 })
113 reload(ctx, 'victoriametrics')
33 114
34 ctx.run(updateConfigmapCmd("rules_expected_hosts.yaml")) 115 # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem
35 ctx.run(updateConfigmapCmd("rules_expected_nodes.yaml")) 116 # reload(ctx, 'vmalert')
36 ctx.run(updateConfigmapCmd("rules_k8s.yaml")) 117 ctx.run('kubectl rollout restart deploy/vmalert')
37 ctx.run(updateConfigmapCmd("rules_main.yaml"))
38 reload(ctx, 'vmalert')
39 118
40 @task 119 @task
41 def build_config(ctx): 120 def build_config(ctx):
42 with open('rules/build/expected_hosts.yaml', 'w') as out: 121 with open('rules/build/expected_hosts.yaml', 'w') as out:
43 out.write(hostsExpectedOnline(ctx)) 122 out.write(hostsExpectedOnline(ctx))