# HG changeset patch # User drewp@bigasterisk.com # Date 1689827290 25200 # Node ID eb1de82c93aa43e7f6366e098d6b17c7bfe62547 # Parent d39a8038227bc6f454999e1b96b7fab63a5b40f5 refactor the merging of all the groups diff -r d39a8038227b -r eb1de82c93aa alert_rules.py --- a/alert_rules.py Wed Jul 19 21:27:46 2023 -0700 +++ b/alert_rules.py Wed Jul 19 21:28:10 2023 -0700 @@ -106,7 +106,7 @@ ] -def allRules(): +def allRules(ctx): return { "groups": [ { @@ -114,6 +114,7 @@ "interval": "1m", "rules": k8sRules(), }, + expectedK8sNodesGroup(ctx), # # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name { @@ -269,7 +270,7 @@ }, ], }, - ] + ] + hostsExpectedOnline(ctx)['groups'] } @@ -281,26 +282,19 @@ return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py") -def expectedK8sNodes(ctx): +def expectedK8sNodesGroup(ctx): getNode = _runJson(ctx, "kubectl get node -o json") hosts = [item["metadata"]["name"] for item in getNode["items"]] optionalHosts = {'slash'} return { - "groups": [ - { - "name": "k8s_expected_nodes", - "rules": [ - { - "alert": "kube_node_log_size_report_" + h, - "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' - % h, - "for": "1h", - "annotations": { - "summary": f"no recent k8s log size report from host {h}" - }, - } - for h in hosts if not h in optionalHosts - ], - } - ] + "name": + "k8s_expected_nodes", + "rules": [{ + "alert": "kube_node_log_size_report_" + h, + "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' % h, + "for": "1h", + "annotations": { + "summary": f"no recent k8s log size report from host {h}" + }, + } for h in hosts if h not in optionalHosts], } diff -r d39a8038227b -r eb1de82c93aa tasks.py --- a/tasks.py Wed Jul 19 21:27:46 2023 -0700 +++ b/tasks.py Wed Jul 19 21:28:10 2023 -0700 @@ -11,9 +11,7 @@ @task def push_config(ctx): configObj = scrapeConfig(ctx) - rulesObj = alert_rules.allRules() - rulesObj["groups"] += alert_rules.expectedK8sNodes(ctx)["groups"] - rulesObj["groups"] += alert_rules.hostsExpectedOnline(ctx)["groups"] + rulesObj = alert_rules.allRules(ctx) replaceCmap("victoriametrics-config", {"scrape_main": configObj, "rules": rulesObj})