Mercurial > code > home > repos > victoriametrics
annotate tasks.py @ 16:73d23fdfee8e
change server to ditto
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 23 Jun 2023 23:53:24 -0700 |
| parents | 2c37fab420da |
| children | f5777b65f035 |
| rev | line source |
|---|---|
| 10 | 1 import json |
| 2 import sys | |
| 3 import tempfile | |
| 4 import time | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
5 import urllib.request |
| 10 | 6 from pathlib import Path |
| 7 from typing import Dict | |
| 8 | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
9 from invoke import task |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
10 |
| 12 | 11 sys.path.append('/usr/lib/python3/dist-packages/') |
| 12 import yaml | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
13 |
| 12 | 14 _tfs = [] |
| 15 | |
| 16 | |
| 17 def saveTmp(text): | |
| 18 tf = tempfile.NamedTemporaryFile(mode='wt') | |
| 19 _tfs.append(tf) | |
| 20 tf.write(text) | |
| 21 tf.flush() | |
| 22 return Path(tf.name) | |
| 23 | |
| 24 | |
| 25 def writeConfigmap(ctx, files: Dict[str, Path]): | |
| 26 arg = ','.join(f'{k}={v}' for k, v in files.items()) | |
| 27 ctx.run( | |
| 28 f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -' | |
| 29 ) | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
30 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
31 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
32 def reload(ctx, svc): |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
33 host = ctx.run(f'khost {svc}').stdout |
| 12 | 34 path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc] |
| 35 reload_url = f'http://{host}{path}-/reload' | |
| 36 print(f'reload with POST {reload_url}') | |
| 37 for workaround in [1]: | |
| 38 print(' -> status', | |
| 39 urllib.request.urlopen(reload_url, data=b'unused').status) | |
| 40 time.sleep(0) | |
| 41 | |
| 42 | |
| 43 | |
| 10 | 44 def hostsExpectedOnline(ctx): |
| 45 return ctx.run( | |
| 46 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout | |
| 47 | |
| 48 | |
| 12 | 49 def expectedK8sNodes(ctx): |
| 50 getNode = json.loads(ctx.run("kubectl get node -o json").stdout) | |
| 51 hosts = [item['metadata']['name'] for item in getNode['items']] | |
| 52 return yaml.dump({ | |
| 53 'groups': [{ | |
| 54 'name': | |
| 55 'k8s_expected_nodes', | |
| 56 'rules': [{ | |
| 57 'alert': | |
| 58 'kube_node_log_size_report_' + h, | |
| 59 'expr': | |
| 60 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' | |
| 61 % h, | |
| 62 'for': | |
| 63 '1h', | |
| 64 'annotations': { | |
| 65 'summary': f"no recent k8s log size report from host {h}" | |
| 66 } | |
| 67 } for h in hosts] | |
| 68 }] | |
| 69 }) | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
70 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
71 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
72 @task |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
73 def sync_config(ctx): |
| 12 | 74 config = Path('config') |
| 75 for workaround in [1]: | |
| 76 writeConfigmap( | |
| 77 ctx, { | |
| 78 # 'scrape_ssl.yaml': saveTmp(httpsCertProber()), | |
| 79 'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)), | |
| 80 'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)), | |
| 81 }) | |
| 82 reload(ctx, 'victoriametrics') | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
83 |
| 12 | 84 # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem |
| 85 # reload(ctx, 'vmalert') | |
| 86 ctx.run('kubectl rollout restart deploy/vmalert') | |
| 10 | 87 |
| 88 @task | |
| 89 def build_config(ctx): | |
| 90 with open('rules/build/expected_hosts.yaml', 'w') as out: | |
| 91 out.write(hostsExpectedOnline(ctx)) |