Mercurial > code > home > repos > victoriametrics
annotate alert_rules.py @ 87:598e75e8e2d7
pdm lock
| author | drewp@bigasterisk.com |
|---|---|
| date | Mon, 26 Aug 2024 16:32:35 -0700 |
| parents | ce7c4a918832 |
| children |
| rev | line source |
|---|---|
| 23 | 1 """ |
| 2 pdm run invoke push-config | |
| 3 | |
| 4 docs: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ | |
| 5 "Whenever the alert expression results in one or more vector | |
| 6 elements at a given point in time, the alert counts as active for | |
| 7 these elements' label sets." | |
| 8 also https://www.metricfire.com/blog/top-5-prometheus-alertmanager-gotchas/#Missing-metrics | |
| 9 | |
| 10 """ | |
| 11 | |
| 12 import json | |
| 13 | |
| 14 | |
| 49 | 15 def pomRules(): |
| 16 return [ | |
| 17 { | |
| 18 "alert": "frequent_upstream_connect_failures", | |
|
51
df44473de6a1
let connection failure alerts finish sooner
drewp@bigasterisk.com
parents:
49
diff
changeset
|
19 "expr": "max_over_time(rate(sum by (envoy_cluster_name) (envoy_cluster_upstream_cx_connect_fail))[1h]) > 0" |
| 49 | 20 }, |
| 21 { | |
| 22 "alert": "high_logging_pomerium", | |
| 23 "for": "3h", | |
| 24 "labels": { | |
| 25 "severity": "waste" | |
| 26 }, | |
| 82 | 27 "expr": 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes{container="pomerium"}[1h])) > 50k', |
| 49 | 28 "annotations": { |
| 29 "summary": "high log output rate" | |
| 30 }, | |
| 31 }, | |
| 32 ] | |
| 33 | |
| 34 | |
| 23 | 35 def k8sRules(): |
| 36 # from https://awesome-prometheus-alerts.grep.to/rules.html | |
| 37 return [ | |
| 38 { | |
| 34 | 39 "alert": "metricsTargetMissing", |
| 40 "expr": 'up{job!~"cm-acme-.*"} == 0', | |
| 36 | 41 'for': '10m', |
| 31 | 42 "labels": { |
| 43 "severity": "critical" | |
| 44 }, | |
| 23 | 45 "annotations": { |
| 34 | 46 "summary": "metrics target missing (instance {{ $labels.instance }})", |
| 47 "description": "A metrics target has disappeared. An exporter might be crashed.\n VALUE = {{ $value }}", | |
| 23 | 48 }, |
| 49 }, | |
| 50 { | |
| 51 "alert": "KubernetesMemoryPressure", | |
| 52 "expr": 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1', | |
| 53 "for": "2m", | |
| 31 | 54 "labels": { |
| 55 "severity": "critical" | |
| 56 }, | |
| 23 | 57 "annotations": { |
| 58 "summary": "Kubernetes memory pressure (instance {{ $labels.instance }})", | |
| 59 "description": "{{ $labels.node }} has MemoryPressure condition\n VALUE = {{ $value }}", | |
| 60 }, | |
| 61 }, | |
| 62 { | |
| 63 "alert": "KubernetesDiskPressure", | |
| 64 "expr": 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1', | |
| 65 "for": "2m", | |
| 31 | 66 "labels": { |
| 67 "severity": "critical" | |
| 68 }, | |
| 23 | 69 "annotations": { |
| 70 "summary": "Kubernetes disk pressure (instance {{ $labels.instance }})", | |
| 71 "description": "{{ $labels.node }} has DiskPressure condition\n VALUE = {{ $value }}", | |
| 72 }, | |
| 73 }, | |
| 74 { | |
| 75 "alert": "KubernetesOutOfDisk", | |
| 76 "expr": 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1', | |
| 77 "for": "2m", | |
| 31 | 78 "labels": { |
| 79 "severity": "critical" | |
| 80 }, | |
| 23 | 81 "annotations": { |
| 82 "summary": "Kubernetes out of disk (instance {{ $labels.instance }})", | |
| 83 "description": "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}", | |
| 84 }, | |
| 85 }, | |
| 86 { | |
| 87 "alert": "KubernetesJobFailed", | |
| 88 "expr": "kube_job_status_failed > 0", | |
| 31 | 89 "labels": { |
| 90 "severity": "warning" | |
| 91 }, | |
| 23 | 92 "annotations": { |
| 93 "summary": "Kubernetes Job failed (instance {{ $labels.instance }})", | |
| 94 "description": "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n VALUE = {{ $value }}", | |
| 95 }, | |
| 96 }, | |
| 97 { | |
| 98 "alert": "KubernetesPodCrashLooping", | |
| 99 "expr": "increase(kube_pod_container_status_restarts_total[1m]) > 3", | |
| 100 "for": "2m", | |
| 31 | 101 "labels": { |
| 102 "severity": "warning" | |
| 103 }, | |
| 23 | 104 "annotations": { |
| 105 "summary": "Kubernetes pod crash looping (instance {{ $labels.instance }})", | |
| 106 "description": "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}", | |
| 107 }, | |
| 108 }, | |
| 109 { | |
| 48 | 110 "alert": "KubernetesClientCertificateExpiresNextWeek", |
| 111 "expr": 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60', | |
| 31 | 112 "labels": { |
| 113 "severity": "warning" | |
| 114 }, | |
| 23 | 115 "annotations": { |
| 116 "summary": "Kubernetes client certificate expires next week (instance {{ $labels.instance }})", | |
| 117 "description": "A client certificate used to authenticate to the apiserver is expiring next week.\n VALUE = {{ $value }}", | |
| 118 }, | |
| 119 }, | |
| 120 { | |
| 121 "alert": "container_waiting", | |
| 34 | 122 "expr": "sum by (namespace, pod, container)(kube_pod_container_status_waiting!=0)", |
| 123 "annotations": { | |
| 124 "description": '', | |
| 125 "dashboard": "https://bigasterisk.com/k/clusters/local/namespaces/{{ $labels.namespace }}/pods/{{ $labels.pod }}", | |
| 126 }, | |
| 81 | 127 "for": "10m", |
| 23 | 128 }, |
| 129 ] | |
| 130 | |
| 131 | |
|
32
eb1de82c93aa
refactor the merging of all the groups
drewp@bigasterisk.com
parents:
31
diff
changeset
|
132 def allRules(ctx): |
| 23 | 133 return { |
| 134 "groups": [ | |
| 135 { | |
| 136 "name": "k8s", | |
|
28
e114edff93dc
more explicit intervals. try to get a single day of notification out of a disk err increase
drewp@bigasterisk.com
parents:
27
diff
changeset
|
137 "interval": "1m", |
| 23 | 138 "rules": k8sRules(), |
| 139 }, | |
| 140 { | |
| 49 | 141 "name": "pomerium_proxy", |
| 142 "interval": "1m", | |
| 143 "rules": pomRules(), | |
| 144 }, | |
| 145 { | |
| 31 | 146 "name": |
| 147 "Outages", | |
| 148 "interval": | |
| 149 "1m", | |
| 23 | 150 "rules": [ |
| 151 { | |
| 152 "alert": "powereagleStalled", | |
| 153 "expr": "rate(house_power_w[100m]) == 0", | |
| 154 "for": "0m", | |
| 31 | 155 "labels": { |
| 156 "severity": "losingData" | |
| 157 }, | |
| 23 | 158 "annotations": { |
| 159 "summary": "power eagle data stalled", | |
| 160 "description": "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs", | |
| 161 }, | |
| 162 }, | |
| 163 { | |
| 164 "alert": "powereagleAbsent", | |
| 165 "expr": "absent_over_time(house_power_w[5m])", | |
| 166 "for": "2m", | |
| 31 | 167 "labels": { |
| 168 "severity": "losingData" | |
| 169 }, | |
| 23 | 170 "annotations": { |
| 171 "summary": "power eagle data missing", | |
| 172 "description": "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs", | |
| 173 }, | |
| 174 }, | |
| 175 { | |
| 176 "alert": "absent_zigbee", | |
| 177 "expr": 'absent(container_last_seen{container="zigbee2mqtt"})', | |
| 81 | 178 "for": "10m", |
| 23 | 179 }, |
| 180 { | |
| 181 "alert": "net_routes_sync", | |
| 182 "expr": 'rate(starlette_request_duration_seconds_count{app_name="net_routes",path="/routes"}[5m]) < 1/70', | |
| 183 "for": "10m", | |
| 31 | 184 "labels": { |
| 185 "severity": "houseUsersAffected" | |
| 186 }, | |
| 23 | 187 "annotations": { |
| 80 | 188 "summary": "net_routes is not getting regular updates from net_routes_input", |
| 23 | 189 }, |
| 190 }, | |
| 191 ], | |
| 192 }, | |
| 193 { | |
| 48 | 194 "name": "disk_errs", |
| 195 "interval": "2d", | |
| 31 | 196 "rules": [{ |
| 36 | 197 "alert": "zpool_device_error_increase", |
| 198 "labels": { | |
| 199 "severity": "warning" | |
| 200 }, | |
| 201 "expr": 'increase(zpool_device_error_count[3d]) > 0', | |
| 202 }, { | |
| 31 | 203 "alert": "zpool_device_error_count", |
| 204 "labels": { | |
| 205 "severity": "warning" | |
|
28
e114edff93dc
more explicit intervals. try to get a single day of notification out of a disk err increase
drewp@bigasterisk.com
parents:
27
diff
changeset
|
206 }, |
| 36 | 207 "expr": 'zpool_device_error_count > 0', |
| 31 | 208 }], |
|
28
e114edff93dc
more explicit intervals. try to get a single day of notification out of a disk err increase
drewp@bigasterisk.com
parents:
27
diff
changeset
|
209 }, |
|
e114edff93dc
more explicit intervals. try to get a single day of notification out of a disk err increase
drewp@bigasterisk.com
parents:
27
diff
changeset
|
210 { |
| 49 | 211 "name": "lighting", |
| 212 "interval": "5m", | |
| 213 "rules": [{ | |
| 214 "alert": "light_bridge_no_mqtt", | |
| 215 "expr": 'mqtt_connected{job="light-bridge"} != 1', | |
| 216 }], | |
| 217 }, | |
| 218 { | |
| 31 | 219 "name": |
| 34 | 220 "front_door", |
| 221 "interval": | |
| 222 "5m", | |
| 223 "rules": [ | |
| 224 { | |
| 36 | 225 "alert": "front_door_reader_esp32_no_mqtt", |
| 226 'expr': 'hw_connected{job="fingerprint"} < 1', | |
| 227 "annotations": { | |
| 228 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 229 }, | |
| 230 }, | |
| 231 { | |
| 48 | 232 "alert": "front_door_reader_svc_down", |
| 36 | 233 'expr': 'up{job="fingerprint"} < 1', |
| 234 "annotations": { | |
| 235 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 236 }, | |
| 237 }, | |
| 238 { | |
| 48 | 239 "alert": "front_door_reader_svc_reader_no_mqtt", |
| 36 | 240 'expr': 'mqtt_connected{job="fingerprint"} < 1', |
| 241 "annotations": { | |
| 242 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 243 }, | |
| 34 | 244 }, |
| 245 { | |
| 36 | 246 "alert": "front_door_lock_svc_down", |
| 247 'expr': 'up{job="front-door-lock"} < 1', | |
| 248 "annotations": { | |
| 249 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 250 }, | |
| 34 | 251 }, |
| 36 | 252 { |
| 253 "alert": "front_door_lock_svc_no_mqtt", | |
| 254 'expr': 'mqtt_connected{job="front-door-lock"} < 1', | |
| 255 "annotations": { | |
| 256 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 257 }, | |
| 258 }, | |
| 259 { | |
| 260 "alert": "front_door_lock_esp32_no_mqtt", | |
| 261 'expr': 'hw_connected{job="front-door-lock"} < 1', | |
| 262 "annotations": { | |
| 263 "summary": "see https://bigasterisk.com/front-door-lock/" | |
| 264 }, | |
| 265 }, | |
| 86 | 266 { |
| 267 "alert": "front_door_display_low_traffic", | |
| 268 'expr': 'rate(lan_bytes_sent_from_total{mac="00:e0:4c:68:0a:19"}[3m]) < 20', | |
| 269 "annotations": { | |
| 270 }, | |
| 271 }, | |
| 36 | 272 ], |
| 34 | 273 }, |
| 274 { | |
| 49 | 275 "name": |
| 276 "net_routes", | |
| 277 "interval": | |
| 278 "5m", | |
| 279 "rules": [ | |
| 280 { | |
| 281 "alert": "no_house_ip_service", | |
| 282 "expr": 'absent(kube_service_spec_external_ip{service="net-route-input-allowed",external_ip="10.2.0.133"})' | |
| 283 }, | |
| 284 { | |
| 285 "alert": "no_net_routes_running", | |
| 286 "expr": 'absent(python_info{job="net-routes"})' | |
| 287 }, | |
| 288 { | |
| 289 "alert": "allowed_check_never_returned_200", | |
| 290 'expr': 'starlette_requests_total{app_name="net_route_input",method="GET",path="/api/allowed",status_code="200"} < 1' | |
| 291 }, | |
| 292 { | |
| 293 "alert": "allowed_check_never_returned_403", | |
| 294 'expr': 'starlette_requests_total{app_name="net_route_input",method="GET",path="/api/allowed",status_code="403"} < 1' | |
| 295 }, | |
| 296 { | |
| 297 'alert': 'net_route_input_eval_cal_loop_is_down', | |
| 298 'expr': 'eval_cal_up!=1' | |
| 299 }, | |
| 300 { | |
| 301 'alert': 'net_route_input_mongo_loop_is_down', | |
| 302 'expr': 'mongo_to_net_routes_up!=1' | |
| 303 }, | |
| 304 { | |
| 305 'alert': 'gcalendarwatch_hasnt_succeeded_on_any_currentEvents_requests', | |
| 306 'expr': 'starlette_requests_total{app_name="gcalendarwatch",method="GET",path="/graph/currentEvents",status_code="200"} < 1' | |
| 307 }, | |
| 308 { | |
| 309 'alert': 'gcalendarwatch_current_events_loop_is_down', | |
| 310 'expr': 'current_events_up != 1' | |
| 311 }, | |
| 312 ], | |
| 313 }, | |
| 314 { | |
|
41
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
315 "name": "http", |
|
37
6e27d280b598
watch https cert ages (testing with 45d)
drewp@bigasterisk.com
parents:
36
diff
changeset
|
316 "interval": "1h", |
|
41
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
317 'rules': [ |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
318 { |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
319 'alert': 'old_https_certs', |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
320 'expr': 'min by (source) (x509_cert_enddate - now())/86400 < 15', |
| 48 | 321 }, |
| 322 { | |
|
41
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
323 'alert': 'high_500_response_rate', |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
324 'expr': 'avg_over_time(rate(sum by (envoy_cluster_name) (envoy_cluster_internal_upstream_rq_xx{envoy_response_code_class="5"})[20m])) > 0.02', |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
325 }, |
|
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
40
diff
changeset
|
326 ], |
|
37
6e27d280b598
watch https cert ages (testing with 45d)
drewp@bigasterisk.com
parents:
36
diff
changeset
|
327 }, |
|
6e27d280b598
watch https cert ages (testing with 45d)
drewp@bigasterisk.com
parents:
36
diff
changeset
|
328 { |
| 42 | 329 "name": "ping", |
| 330 "interval": "1m", | |
| 331 "rules": [{ | |
| 332 "alert": "ping_failed", | |
| 333 "expr": 'max_over_time(probe_success{job="ping"}[1m]) < 1', | |
| 82 | 334 "for": "10m", |
| 42 | 335 }] |
| 336 }, | |
| 337 { | |
| 34 | 338 "name": |
| 31 | 339 "alerts", |
| 23 | 340 "rules": [ |
| 341 { | |
| 342 "alert": "kube_node_status_bad_condition", | |
| 343 "for": "2h", | |
| 31 | 344 "labels": { |
| 345 "severity": "warning" | |
| 346 }, | |
| 23 | 347 "expr": 'kube_node_status_condition{condition=~".*Pressure",status="true"} > 0', |
| 348 }, | |
| 349 { | |
| 350 "alert": "housePower", | |
|
28
e114edff93dc
more explicit intervals. try to get a single day of notification out of a disk err increase
drewp@bigasterisk.com
parents:
27
diff
changeset
|
351 "for": "1h", |
| 31 | 352 "labels": { |
| 353 "severity": "waste" | |
| 354 }, | |
| 23 | 355 "expr": "house_power_w > 4000", |
| 31 | 356 "annotations": { |
| 357 "summary": "house power usage over 4KW" | |
| 358 }, | |
| 23 | 359 }, |
| 360 { | |
| 361 "alert": "host_root_fs_space_low", | |
| 362 "for": "20m", | |
| 31 | 363 "labels": { |
| 364 "severity": "warning" | |
| 365 }, | |
| 82 | 366 "expr": 'disk_used_percent{path="/"} > 85', |
| 23 | 367 }, |
| 368 { | |
| 369 "alert": "zpool_space_low", | |
| 370 "for": "20m", | |
| 31 | 371 "labels": { |
| 372 "severity": "warning" | |
| 373 }, | |
| 23 | 374 "expr": 'last_over_time(zfs_pool_free_bytes{pool="stor7"}[1h]) < 100G', |
| 375 }, | |
| 376 { | |
| 377 "alert": "disk_week_incr", | |
| 378 "for": "20m", | |
| 31 | 379 "labels": { |
| 380 "severity": "warning" | |
| 381 }, | |
| 23 | 382 "expr": 'round(increase(disk_used{path=~"/my/.*"}[1d])/1M) > 5000', |
| 31 | 383 "annotations": { |
| 384 "summary": "high mb/week on zfs dir" | |
| 385 }, | |
| 23 | 386 }, |
| 387 { | |
| 388 "alert": "high_logging", | |
| 31 | 389 "for": "3h", |
| 390 "labels": { | |
| 391 "severity": "waste" | |
| 392 }, | |
|
76
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
67
diff
changeset
|
393 "expr": 'sum by (namespace, pod, container) (rate(kubelet_container_log_filesystem_used_bytes{container!="pomerium"}[3h])) > 10k', |
| 31 | 394 "annotations": { |
| 395 "summary": "high log output rate" | |
| 396 }, | |
| 23 | 397 }, |
| 398 { | |
| 399 "alert": "stale_process", | |
| 400 "for": "1d", | |
| 31 | 401 "labels": { |
| 402 "severity": "dataRisk" | |
| 403 }, | |
| 23 | 404 "expr": "round((time() - filestat_modification_time/1e9) / 86400) > 14", |
| 31 | 405 "annotations": { |
| 406 "summary": "process time is old" | |
| 407 }, | |
| 23 | 408 }, |
| 409 { | |
| 410 "alert": "starlette", | |
| 411 "for": "1m", | |
| 31 | 412 "labels": { |
| 413 "severity": "fix" | |
| 414 }, | |
| 23 | 415 "expr": 'starlette_request_duration_seconds_created{app_name="starlette"}', |
| 31 | 416 "annotations": { |
| 417 "summary": "set starlette app name" | |
| 418 }, | |
| 23 | 419 }, |
| 420 { | |
| 421 "alert": "ssl_certs_expiring_soon", | |
| 422 "expr": "min((min_over_time(probe_ssl_earliest_cert_expiry[1d])-time())/86400) < 10", | |
| 31 | 423 "labels": { |
| 424 "severity": "warning" | |
| 425 }, | |
| 23 | 426 "annotations": { |
| 427 "summary": "cert expiring soon. See https://bigasterisk.com/grafana/d/z1YtDa3Gz/certs?orgId=1\nVALUE = {{ $value }}" | |
| 428 }, | |
| 429 }, | |
| 430 ], | |
| 431 }, | |
|
32
eb1de82c93aa
refactor the merging of all the groups
drewp@bigasterisk.com
parents:
31
diff
changeset
|
432 ] + hostsExpectedOnline(ctx)['groups'] |
| 23 | 433 } |
| 434 | |
| 435 | |
| 436 def _runJson(ctx, cmd): | |
| 437 return json.loads(ctx.run(cmd, hide="stdout").stdout) | |
| 438 | |
| 439 | |
| 440 def hostsExpectedOnline(ctx): | |
| 441 return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py") |