Mercurial > code > home > repos > victoriametrics

comparison tasks.py @ 22:cd115f1ca2a8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use configmaps and a special pod refresh trick
author drewp@bigasterisk.com
date Sat, 24 Jun 2023 23:02:04 -0700
parents f5777b65f035
children eec015e90818
comparison
equal deleted inserted replaced
21:10127391f6f3 22:cd115f1ca2a8
1 import json 1 import json
2 import sys
3 import tempfile
4 import time
5 import urllib.request
6 from pathlib import Path
7 from typing import Dict
8 2
9 import yaml 3 import yaml
10 from invoke import task 4 from invoke import task
11 from kubernetes import config 5 from kubernetes import config
12 6
13 from k8s_ops import firstPodName, replaceCmap, refreshPodCmaps 7 from k8s_ops import firstPodName, replaceCmap, refreshPodCmaps
8 import alert_rules
14 9
15 _tfs = []
16 config.load_kube_config() 10 config.load_kube_config()
17 11
18 12
19 def saveTmp(text): 13 @task
20 tf = tempfile.NamedTemporaryFile(mode='wt') 14 def push_config(ctx):
21 _tfs.append(tf) 15 configObj = scrapeConfig(ctx)
22 tf.write(text) 16 rulesObj = alert_rules.allRules()
23 tf.flush() 17 rulesObj["groups"] += alert_rules.expectedK8sNodes(ctx)["groups"]
24 return Path(tf.name) 18 rulesObj["groups"] += alert_rules.hostsExpectedOnline(ctx)["groups"]
25 19
20 replaceCmap("victoriametrics-config", {"scrape_main": configObj, "rules": rulesObj})
26 21
27 def writeConfigmap(ctx, files: Dict[str, Path]): 22 refreshPodCmaps(firstPodName("app=victoriametrics"))
28 arg = ','.join(f'{k}={v}' for k, v in files.items()) 23 refreshPodCmaps(firstPodName("app=vmalert"))
29 ctx.run(
30 f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -'
31 )
32 24
33 25 # If the VM reloader isn't fast enough, we could do this too:
34 def reload(ctx, svc): 26 # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod')
35 host = ctx.run(f'khost {svc}').stdout
36 path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc]
37 reload_url = f'http://{host}{path}-/reload'
38 print(f'reload with POST {reload_url}')
39 for workaround in [1]:
40 print(' -> status',
41 urllib.request.urlopen(reload_url, data=b'unused').status)
42 time.sleep(0)
43
44
45 def hostsExpectedOnline(ctx):
46 return ctx.run(
47 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout
48
49
50 def expectedK8sNodes(ctx):
51 getNode = json.loads(ctx.run("kubectl get node -o json").stdout)
52 hosts = [item['metadata']['name'] for item in getNode['items']]
53 return yaml.dump({
54 'groups': [{
55 'name':
56 'k8s_expected_nodes',
57 'rules': [{
58 'alert':
59 'kube_node_log_size_report_' + h,
60 'expr':
61 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})'
62 % h,
63 'for':
64 '1h',
65 'annotations': {
66 'summary': f"no recent k8s log size report from host {h}"
67 }
68 } for h in hosts]
69 }]
70 })
71
72
73 @task
74 def sync_config(ctx):
75 config = Path('config')
76 for workaround in [1]:
77 writeConfigmap(
78 ctx, {
79 # 'scrape_ssl.yaml': saveTmp(httpsCertProber()),
80 'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)),
81 'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)),
82 })
83 reload(ctx, 'victoriametrics')
84
85 # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem
86 # reload(ctx, 'vmalert')
87 ctx.run('kubectl rollout restart deploy/vmalert')
88
89
90 @task
91 def build_config(ctx):
92 with open('rules/build/expected_hosts.yaml', 'w') as out:
93 out.write(hostsExpectedOnline(ctx))
94
95
96 # --------------------------
97 27
98 28
99 def scrapeConfig(ctx): 29 def scrapeConfig(ctx):
100 return yaml.load(open('config/scrape_main.yaml'), yaml.FullLoader) 30 return yaml.load(open("config/scrape_main.yaml"), yaml.FullLoader)
101
102
103 @task
104 def updateScrapes(ctx):
105 configObj = scrapeConfig(ctx)
106
107 replaceCmap('victoriametrics-config', configObj)
108
109 refreshPodCmaps(firstPodName('app=victoriametrics'))
110
111 # If the VM reloader isn't fast enough, we could do this too:
112 # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod')