Mercurial > code > home > repos > victoriametrics

comparison rules/rules_main.yaml @ 9:17db5e8e7a2f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
big rules and scrape config updates
author drewp@bigasterisk.com
date Sun, 04 Dec 2022 02:08:08 -0800
parents config/rules_main.yaml@1eb6e6a2b9b6
children b6720e379d5b
comparison
equal deleted inserted replaced
8:e393b24f0e01 9:17db5e8e7a2f
1 groups:
2 # docs: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/
3 # "Whenever the alert expression results in one or more vector
4 # elements at a given point in time, the alert counts as active for
5 # these elements' label sets."
6
7 # also https://www.metricfire.com/blog/top-5-prometheus-alertmanager-gotchas/#Missing-metrics
8 #
9 # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name
10
11 # - name: webcam
12 # rules:
13 # waiting for twinscam revival
14 # - alert: twinscam_not_reporting
15 # expr: absent(cam_pipeline_state{job="webcam-record-twinscam"})
16 # for: 2m
17 # labels:
18 # severity: losingData
19 # annotations:
20 # summary: "webcam-record-twinscam is not reporting metrics {{ $labels }}"
21
22 # - alert: cam_garagehall_not_reporting
23 # expr: absent(cam_pipeline_state{job="webcam-record-garagehall"})
24 # for: 2m
25 # labels:
26 # severity: losingData
27 # annotations:
28 # # summary: "webcam-record-garagehall is not reporting metrics {{ $labels }}"
29
30 # - alert: cam_pipeline_stopped
31 # expr: sum without (instance) (cam_pipeline_state{cam_pipeline_state="playing"}) < 1
32 # for: 10m
33 # labels:
34 # severity: losingData
35 # annotations:
36 # summary: "webcam-record gst pipeline is not state=playing {{ $labels }}"
37
38 # - alert: cam_not_advancing
39 # expr: rate(cam_stream_bytes{element="splitmux"}[3m]) < 0.2
40 # for: 10m
41 # labels:
42 # severity: losingData
43 # annotations:
44 # summary: "cam output bytes is advancing too slowly. {{ $labels }}"
45
46 # - alert: webcam_indexer_stalled
47 # expr: rate(webcam_indexer_update_count{job="webcam-indexer"}[5m]) < .01
48 # for: 10m
49 # labels:
50 # severity: webcamUsersAffected
51 # annotations:
52 # summary: "webcam indexer update loop is stalled"
53
54 - name: Outages
55 rules:
56 - alert: powereagleStalled
57 expr: rate(house_power_w[100m]) == 0
58 for: 0m
59 labels:
60 severity: losingData
61 annotations:
62 summary: "power eagle data stalled"
63 description: "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs"
64
65 - alert: powereagleAbsent
66 expr: absent_over_time(house_power_w[5m])
67 for: 2m
68 labels:
69 severity: losingData
70 annotations:
71 summary: "power eagle data missing"
72 description: "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs"
73
74 # - alert: wifi_scrape_errors
75 # expr: rate(poll_errors_total{job="wifi"}[2m]) > .1
76 # labels:
77 # severity: houseUsersAffected
78 # annotations:
79 # summary: "errors getting wifi users list"
80
81 # - alert: absent_mitmproxy
82 # expr: absent(process_resident_memory_bytes{job="mitmproxy"})
83 # labels:
84 # severity: houseUsersAffected
85 # annotations:
86 # summary: "mitmproxy metrics not responding. See https://bigasterisk.com/grafana/d/ix3hMAdMk/webfilter?orgId=1&from=now-12h&to=now and https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/mitmproxy (metrics actually come from webfilter.py plugin)"
87
88 - alert: absent_zigbee_dash
89 expr: absent(container_last_seen{container="zigbee2mqtt-dash"})
90
91 - alert: net_routes_sync
92 expr: rate(starlette_request_duration_seconds_count{app_name="net_routes",path="/routes"}[5m]) < 1/70
93 for: 10m
94 labels:
95 severity: houseUsersAffected
96 annotations:
97 summary: "net_routes is not getting regular updates"
98
99
100 - name: alerts
101 rules:
102 - {alert: housePower, for: 24h, labels: {severity: waste}, expr: "house_power_w > 4000", annotations: {summary: "house power usage over 3KW {{ $labels }}"}}
103 - {alert: disk1, for: 20m, labels: {severity: warning}, expr: 'disk_free{path=~"/(d[1-9])?"} < 20G', annotations: {summary: "low disk_free {{ $labels }}"}}
104 - {alert: disk2, for: 20m, labels: {severity: warning}, expr: 'disk_free{path="/stor6/my"} < 100G', annotations: {summary: "low disk_free {{ $labels }}"}}
105 - {alert: disk3, for: 20m, labels: {severity: warning}, expr: 'round(increase(disk_used{fstype="zfs",path=~"^/stor6.*"}[1w]) / 1M) > 500', annotations: {summary: "high mb/week on zfs dir {{ $labels }}"}}
106 - {alert: oom, for: 1m, labels: {severity: warning}, expr: 'predict_linear(mem_free[5m], 5m) / 1M < 100', annotations: {summary: "host about to run OOM {{ $labels }}"}}
107 - {alert: high_logging, for: 20m, labels: {severity: waste}, expr: 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes[30m])) > 30000', annotations: {summary: "high log output rate {{ $labels }}"}}
108 - {alert: stale_process, for: 1d, labels: {severity: dataRisk}, expr: 'round((time() - filestat_modification_time/1e9) / 86400) > 14', annotations: {summary: "process time is old {{ $labels }}"}}
109 - {alert: starlette, for: 1m, labels: {severity: fix}, expr: 'starlette_request_duration_seconds_created{app_name="starlette"}', annotations: {summary: "set starlette app name {{ $labels }}"}}
110 - alert: ssl_certs_expiring_soon
111 expr: min((min_over_time(probe_ssl_earliest_cert_expiry[1d])-time())/86400) < 10
112 labels:
113 severity: warning
114 annotations:
115 summary: "cert expiring soon. See https://bigasterisk.com/grafana/d/z1YtDa3Gz/certs?orgId=1\nVALUE = {{ $value }}\n LABELS = {{ $labels }}"