Mercurial > code > home > repos > victoriametrics
annotate tasks.py @ 12:b6720e379d5b
config updates
| author | drewp@bigasterisk.com |
|---|---|
| date | Tue, 14 Mar 2023 20:04:06 -0700 |
| parents | 2023a6ce7bc0 |
| children | 2c37fab420da |
| rev | line source |
|---|---|
| 10 | 1 import json |
| 2 import sys | |
| 3 import tempfile | |
| 4 import time | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
5 import urllib.request |
| 10 | 6 from pathlib import Path |
| 7 from typing import Dict | |
| 8 | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
9 from invoke import task |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
10 |
| 12 | 11 sys.path.append('/usr/lib/python3/dist-packages/') |
| 12 import yaml | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
13 |
| 12 | 14 _tfs = [] |
| 15 | |
| 16 | |
| 17 def saveTmp(text): | |
| 18 tf = tempfile.NamedTemporaryFile(mode='wt') | |
| 19 _tfs.append(tf) | |
| 20 tf.write(text) | |
| 21 tf.flush() | |
| 22 return Path(tf.name) | |
| 23 | |
| 24 | |
| 25 def writeConfigmap(ctx, files: Dict[str, Path]): | |
| 26 arg = ','.join(f'{k}={v}' for k, v in files.items()) | |
| 27 ctx.run( | |
| 28 f'kubectl create configmap victoriametrics-config --from-file {arg} -o yaml --dry-run=client | kubectl apply -f -' | |
| 29 ) | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
30 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
31 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
32 def reload(ctx, svc): |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
33 host = ctx.run(f'khost {svc}').stdout |
| 12 | 34 path = {'victoriametrics': '/m/', 'vmalert': '/'}[svc] |
| 35 reload_url = f'http://{host}{path}-/reload' | |
| 36 print(f'reload with POST {reload_url}') | |
| 37 for workaround in [1]: | |
| 38 print(' -> status', | |
| 39 urllib.request.urlopen(reload_url, data=b'unused').status) | |
| 40 time.sleep(0) | |
| 41 | |
| 42 | |
| 43 def httpsCertProber(): | |
| 44 domains = [] | |
| 45 for line in open( | |
| 46 '/my/doc/ssl/letsencrypt/run.py'): # moved to cert-manager | |
| 47 if line.startswith('update_certs('): | |
| 48 domains.append(line.split("'")[1]) | |
| 49 relabel = { | |
| 50 'relabel_configs': [{ | |
| 51 'source_labels': ['__address__'], | |
| 52 'target_label': '__param_target' | |
| 53 }, { | |
| 54 'source_labels': ['__param_target'], | |
| 55 'target_label': 'instance' | |
| 56 }, { | |
| 57 'target_label': '__address__', | |
| 58 'replacement': 'prober' | |
| 59 }] | |
| 60 } | |
| 61 return yaml.dump( # Note that an included file must skip the scrape_configs toplevel key and just include the list. | |
| 62 [{ | |
| 63 'job_name': 'prober', | |
| 64 'scrape_interval': '24h', | |
| 65 'metrics_path': '/probe', | |
| 66 'params': { | |
| 67 'module': ['https'] | |
| 68 }, | |
| 69 'static_configs': [{ | |
| 70 'targets': domains | |
| 71 }], | |
| 72 } | relabel]) | |
| 73 | |
| 74 | |
| 10 | 75 def hostsExpectedOnline(ctx): |
| 76 return ctx.run( | |
| 77 'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout | |
| 78 | |
| 79 | |
| 12 | 80 def expectedK8sNodes(ctx): |
| 81 getNode = json.loads(ctx.run("kubectl get node -o json").stdout) | |
| 82 hosts = [item['metadata']['name'] for item in getNode['items']] | |
| 83 return yaml.dump({ | |
| 84 'groups': [{ | |
| 85 'name': | |
| 86 'k8s_expected_nodes', | |
| 87 'rules': [{ | |
| 88 'alert': | |
| 89 'kube_node_log_size_report_' + h, | |
| 90 'expr': | |
| 91 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' | |
| 92 % h, | |
| 93 'for': | |
| 94 '1h', | |
| 95 'annotations': { | |
| 96 'summary': f"no recent k8s log size report from host {h}" | |
| 97 } | |
| 98 } for h in hosts] | |
| 99 }] | |
| 100 }) | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
101 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
102 |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
103 @task |
|
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
104 def sync_config(ctx): |
| 12 | 105 config = Path('config') |
| 106 for workaround in [1]: | |
| 107 writeConfigmap( | |
| 108 ctx, { | |
| 109 # 'scrape_ssl.yaml': saveTmp(httpsCertProber()), | |
| 110 'rules_expected_nodes.yaml': saveTmp(expectedK8sNodes(ctx)), | |
| 111 'rules_expected_hosts.yaml': saveTmp(hostsExpectedOnline(ctx)), | |
| 112 }) | |
| 113 reload(ctx, 'victoriametrics') | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
114 |
| 12 | 115 # this reload doesn't get the new config- not sure if it's vmalert bug or k8s cm propogation problem |
| 116 # reload(ctx, 'vmalert') | |
| 117 ctx.run('kubectl rollout restart deploy/vmalert') | |
| 10 | 118 |
| 119 @task | |
| 120 def build_config(ctx): | |
| 121 with open('rules/build/expected_hosts.yaml', 'w') as out: | |
| 122 out.write(hostsExpectedOnline(ctx)) |