Mercurial > code > home > repos > victoriametrics

changeset 36:2bc188c4117a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rules updates, incl front door group
author drewp@bigasterisk.com
date Sun, 15 Oct 2023 18:34:29 -0700
parents 80e275ab2f88
children 6e27d280b598
files alert_rules.py
diffstat 1 files changed, 51 insertions(+), 8 deletions(-) [+]
line wrap: on
line diff
--- a/alert_rules.py	Mon Oct 09 18:51:04 2023 -0700
+++ b/alert_rules.py	Sun Oct 15 18:34:29 2023 -0700
@@ -18,6 +18,7 @@
         {
             "alert": "metricsTargetMissing",
             "expr": 'up{job!~"cm-acme-.*"} == 0',
+            'for': '10m',
             "labels": {
                 "severity": "critical"
             },
@@ -169,14 +170,22 @@
                 ],
             },
             {
-                "name": "disk_errs",
-                "interval": "2d",
+                "name":
+                    "disk_errs",
+                "interval":
+                    "2d",
                 "rules": [{
+                    "alert": "zpool_device_error_increase",
+                    "labels": {
+                        "severity": "warning"
+                    },
+                    "expr": 'increase(zpool_device_error_count[3d]) > 0',
+                }, {
                     "alert": "zpool_device_error_count",
                     "labels": {
                         "severity": "warning"
                     },
-                    "expr": 'increase(zpool_device_error_count[3d]) > 0',
+                    "expr": 'zpool_device_error_count > 0',
                 }],
             },
             {
@@ -186,14 +195,48 @@
                     "5m",
                 "rules": [
                     {
-                        "alert": "service_disconnected_from_mqtt",
-                        "expr": "mqtt_connected < 1"
+                        "alert": "front_door_reader_esp32_no_mqtt",
+                        'expr': 'hw_connected{job="fingerprint"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
+                    },
+                    {
+                        "alert": "fronr_door_reader_svc_down",
+                        'expr': 'up{job="fingerprint"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
+                    },
+                    {
+                        "alert": "fronr_door_reader_svc_reader_no_mqtt",
+                        'expr': 'mqtt_connected{job="fingerprint"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
                     },
                     {
-                        "alert": "esp_not_connected_to_mqtt",
-                        "expr": "hw_connected < 1",
+                        "alert": "front_door_lock_svc_down",
+                        'expr': 'up{job="front-door-lock"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
                     },
-                ]
+                    {
+                        "alert": "front_door_lock_svc_no_mqtt",
+                        'expr': 'mqtt_connected{job="front-door-lock"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
+                    },
+                    {
+                        "alert": "front_door_lock_esp32_no_mqtt",
+                        'expr': 'hw_connected{job="front-door-lock"} < 1',
+                        "annotations": {
+                            "summary": "see https://bigasterisk.com/front-door-lock/"
+                        },
+                    },
+                ],
             },
             {
                 "name":