Mercurial > code > home > repos > victoriametrics

diff alert_rules.py @ 76:009527a145d0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add kube-state-metrics scrape; loosen some high-logging thresholds
author drewp@bigasterisk.com
date Sun, 05 May 2024 18:51:35 -0700
parents adde35eb4773
children 298a6f9f7da7
line wrap: on
line diff
--- a/alert_rules.py	Sat May 04 19:30:56 2024 -0700
+++ b/alert_rules.py	Sun May 05 18:51:35 2024 -0700
@@ -24,7 +24,7 @@
             "labels": {
                 "severity": "waste"
             },
-            "expr": 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes{container="pomerium"}[3h])) > 8k',
+            "expr": 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes{container="pomerium"}[1h])) > 12k',
             "annotations": {
                 "summary": "high log output rate"
             },
@@ -382,7 +382,7 @@
                         "labels": {
                             "severity": "waste"
                         },
-                        "expr": 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes{container!="pomerium"}[3h])) > 4k',
+                        "expr": 'sum by (namespace, pod, container) (rate(kubelet_container_log_filesystem_used_bytes{container!="pomerium"}[3h])) > 10k',
                         "annotations": {
                             "summary": "high log output rate"
                         },