Mercurial > code > home > repos > victoriametrics
changeset 32:eb1de82c93aa
refactor the merging of all the groups
author | drewp@bigasterisk.com |
---|---|
date | Wed, 19 Jul 2023 21:28:10 -0700 |
parents | d39a8038227b |
children | ce7ae4d2b24b |
files | alert_rules.py tasks.py |
diffstat | 2 files changed, 15 insertions(+), 23 deletions(-) [+] |
line wrap: on
line diff
--- a/alert_rules.py Wed Jul 19 21:27:46 2023 -0700 +++ b/alert_rules.py Wed Jul 19 21:28:10 2023 -0700 @@ -106,7 +106,7 @@ ] -def allRules(): +def allRules(ctx): return { "groups": [ { @@ -114,6 +114,7 @@ "interval": "1m", "rules": k8sRules(), }, + expectedK8sNodesGroup(ctx), # # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name { @@ -269,7 +270,7 @@ }, ], }, - ] + ] + hostsExpectedOnline(ctx)['groups'] } @@ -281,26 +282,19 @@ return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py") -def expectedK8sNodes(ctx): +def expectedK8sNodesGroup(ctx): getNode = _runJson(ctx, "kubectl get node -o json") hosts = [item["metadata"]["name"] for item in getNode["items"]] optionalHosts = {'slash'} return { - "groups": [ - { - "name": "k8s_expected_nodes", - "rules": [ - { - "alert": "kube_node_log_size_report_" + h, - "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' - % h, - "for": "1h", - "annotations": { - "summary": f"no recent k8s log size report from host {h}" - }, - } - for h in hosts if not h in optionalHosts - ], - } - ] + "name": + "k8s_expected_nodes", + "rules": [{ + "alert": "kube_node_log_size_report_" + h, + "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' % h, + "for": "1h", + "annotations": { + "summary": f"no recent k8s log size report from host {h}" + }, + } for h in hosts if h not in optionalHosts], }
--- a/tasks.py Wed Jul 19 21:27:46 2023 -0700 +++ b/tasks.py Wed Jul 19 21:28:10 2023 -0700 @@ -11,9 +11,7 @@ @task def push_config(ctx): configObj = scrapeConfig(ctx) - rulesObj = alert_rules.allRules() - rulesObj["groups"] += alert_rules.expectedK8sNodes(ctx)["groups"] - rulesObj["groups"] += alert_rules.hostsExpectedOnline(ctx)["groups"] + rulesObj = alert_rules.allRules(ctx) replaceCmap("victoriametrics-config", {"scrape_main": configObj, "rules": rulesObj})