changeset 32:eb1de82c93aa

refactor the merging of all the groups
author drewp@bigasterisk.com
date Wed, 19 Jul 2023 21:28:10 -0700
parents d39a8038227b
children ce7ae4d2b24b
files alert_rules.py tasks.py
diffstat 2 files changed, 15 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/alert_rules.py	Wed Jul 19 21:27:46 2023 -0700
+++ b/alert_rules.py	Wed Jul 19 21:28:10 2023 -0700
@@ -106,7 +106,7 @@
     ]
 
 
-def allRules():
+def allRules(ctx):
     return {
         "groups": [
             {
@@ -114,6 +114,7 @@
                 "interval": "1m",
                 "rules": k8sRules(),
             },
+            expectedK8sNodesGroup(ctx),
             #
             # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name
             {
@@ -269,7 +270,7 @@
                     },
                 ],
             },
-        ]
+        ] + hostsExpectedOnline(ctx)['groups']
     }
 
 
@@ -281,26 +282,19 @@
     return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py")
 
 
-def expectedK8sNodes(ctx):
+def expectedK8sNodesGroup(ctx):
     getNode = _runJson(ctx, "kubectl get node -o json")
     hosts = [item["metadata"]["name"] for item in getNode["items"]]
     optionalHosts = {'slash'}
     return {
-        "groups": [
-            {
-                "name": "k8s_expected_nodes",
-                "rules": [
-                    {
-                        "alert": "kube_node_log_size_report_" + h,
-                        "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})'
-                        % h,
-                        "for": "1h",
-                        "annotations": {
-                            "summary": f"no recent k8s log size report from host {h}"
-                        },
-                    }
-                    for h in hosts if not h in optionalHosts
-                ],
-            }
-        ]
+        "name":
+            "k8s_expected_nodes",
+        "rules": [{
+            "alert": "kube_node_log_size_report_" + h,
+            "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' % h,
+            "for": "1h",
+            "annotations": {
+                "summary": f"no recent k8s log size report from host {h}"
+            },
+        } for h in hosts if h not in optionalHosts],
     }
--- a/tasks.py	Wed Jul 19 21:27:46 2023 -0700
+++ b/tasks.py	Wed Jul 19 21:28:10 2023 -0700
@@ -11,9 +11,7 @@
 @task
 def push_config(ctx):
     configObj = scrapeConfig(ctx)
-    rulesObj = alert_rules.allRules()
-    rulesObj["groups"] += alert_rules.expectedK8sNodes(ctx)["groups"]
-    rulesObj["groups"] += alert_rules.hostsExpectedOnline(ctx)["groups"]
+    rulesObj = alert_rules.allRules(ctx)
 
     replaceCmap("victoriametrics-config", {"scrape_main": configObj, "rules": rulesObj})