|
23
|
1 """
|
|
|
2 pdm run invoke push-config
|
|
|
3
|
|
|
4 docs: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/
|
|
|
5 "Whenever the alert expression results in one or more vector
|
|
|
6 elements at a given point in time, the alert counts as active for
|
|
|
7 these elements' label sets."
|
|
|
8 also https://www.metricfire.com/blog/top-5-prometheus-alertmanager-gotchas/#Missing-metrics
|
|
|
9
|
|
|
10 """
|
|
|
11
|
|
|
12 import json
|
|
|
13
|
|
|
14 import yaml
|
|
|
15
|
|
|
16
|
|
|
17 def k8sRules():
|
|
|
18 # from https://awesome-prometheus-alerts.grep.to/rules.html
|
|
|
19 return [
|
|
|
20 {
|
|
|
21 "alert": "PrometheusTargetMissing",
|
|
|
22 "expr": "up == 0",
|
|
|
23 "for": "0m",
|
|
|
24 "labels": {"severity": "critical"},
|
|
|
25 "annotations": {
|
|
|
26 "summary": "Prometheus target missing (instance {{ $labels.instance }})",
|
|
|
27 "description": "A Prometheus target has disappeared. An exporter might be crashed.\n VALUE = {{ $value }}",
|
|
|
28 },
|
|
|
29 },
|
|
|
30 {
|
|
|
31 "alert": "KubernetesMemoryPressure",
|
|
|
32 "expr": 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1',
|
|
|
33 "for": "2m",
|
|
|
34 "labels": {"severity": "critical"},
|
|
|
35 "annotations": {
|
|
|
36 "summary": "Kubernetes memory pressure (instance {{ $labels.instance }})",
|
|
|
37 "description": "{{ $labels.node }} has MemoryPressure condition\n VALUE = {{ $value }}",
|
|
|
38 },
|
|
|
39 },
|
|
|
40 {
|
|
|
41 "alert": "KubernetesDiskPressure",
|
|
|
42 "expr": 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1',
|
|
|
43 "for": "2m",
|
|
|
44 "labels": {"severity": "critical"},
|
|
|
45 "annotations": {
|
|
|
46 "summary": "Kubernetes disk pressure (instance {{ $labels.instance }})",
|
|
|
47 "description": "{{ $labels.node }} has DiskPressure condition\n VALUE = {{ $value }}",
|
|
|
48 },
|
|
|
49 },
|
|
|
50 {
|
|
|
51 "alert": "KubernetesOutOfDisk",
|
|
|
52 "expr": 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1',
|
|
|
53 "for": "2m",
|
|
|
54 "labels": {"severity": "critical"},
|
|
|
55 "annotations": {
|
|
|
56 "summary": "Kubernetes out of disk (instance {{ $labels.instance }})",
|
|
|
57 "description": "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}",
|
|
|
58 },
|
|
|
59 },
|
|
|
60 {
|
|
|
61 "alert": "KubernetesJobFailed",
|
|
|
62 "expr": "kube_job_status_failed > 0",
|
|
|
63 "for": "0m",
|
|
|
64 "labels": {"severity": "warning"},
|
|
|
65 "annotations": {
|
|
|
66 "summary": "Kubernetes Job failed (instance {{ $labels.instance }})",
|
|
|
67 "description": "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n VALUE = {{ $value }}",
|
|
|
68 },
|
|
|
69 },
|
|
|
70 {
|
|
|
71 "alert": "KubernetesPodCrashLooping",
|
|
|
72 "expr": "increase(kube_pod_container_status_restarts_total[1m]) > 3",
|
|
|
73 "for": "2m",
|
|
|
74 "labels": {"severity": "warning"},
|
|
|
75 "annotations": {
|
|
|
76 "summary": "Kubernetes pod crash looping (instance {{ $labels.instance }})",
|
|
|
77 "description": "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}",
|
|
|
78 },
|
|
|
79 },
|
|
|
80 {
|
|
|
81 "alert": "KubernetesClientCertificateExpiresNextWeek",
|
|
|
82 "expr": 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60',
|
|
|
83 "for": "0m",
|
|
|
84 "labels": {"severity": "warning"},
|
|
|
85 "annotations": {
|
|
|
86 "summary": "Kubernetes client certificate expires next week (instance {{ $labels.instance }})",
|
|
|
87 "description": "A client certificate used to authenticate to the apiserver is expiring next week.\n VALUE = {{ $value }}",
|
|
|
88 },
|
|
|
89 },
|
|
|
90 {
|
|
|
91 "alert": "container_waiting",
|
|
|
92 "expr": "sum by (container)(kube_pod_container_status_waiting!=0)",
|
|
|
93 "for": "2m",
|
|
|
94 },
|
|
|
95 ]
|
|
|
96
|
|
|
97
|
|
|
98 def allRules():
|
|
|
99 return {
|
|
|
100 "groups": [
|
|
|
101 {
|
|
|
102 "name": "k8s",
|
|
|
103 "rules": k8sRules(),
|
|
|
104 },
|
|
|
105 #
|
|
|
106 # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name
|
|
|
107 {
|
|
|
108 "name": "Outages",
|
|
|
109 "rules": [
|
|
|
110 {
|
|
|
111 "alert": "powereagleStalled",
|
|
|
112 "expr": "rate(house_power_w[100m]) == 0",
|
|
|
113 "for": "0m",
|
|
|
114 "labels": {"severity": "losingData"},
|
|
|
115 "annotations": {
|
|
|
116 "summary": "power eagle data stalled",
|
|
|
117 "description": "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs",
|
|
|
118 },
|
|
|
119 },
|
|
|
120 {
|
|
|
121 "alert": "powereagleAbsent",
|
|
|
122 "expr": "absent_over_time(house_power_w[5m])",
|
|
|
123 "for": "2m",
|
|
|
124 "labels": {"severity": "losingData"},
|
|
|
125 "annotations": {
|
|
|
126 "summary": "power eagle data missing",
|
|
|
127 "description": "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs",
|
|
|
128 },
|
|
|
129 },
|
|
|
130 {
|
|
|
131 "alert": "absent_zigbee",
|
|
|
132 "expr": 'absent(container_last_seen{container="zigbee2mqtt"})',
|
|
|
133 },
|
|
|
134 {
|
|
|
135 "alert": "net_routes_sync",
|
|
|
136 "expr": 'rate(starlette_request_duration_seconds_count{app_name="net_routes",path="/routes"}[5m]) < 1/70',
|
|
|
137 "for": "10m",
|
|
|
138 "labels": {"severity": "houseUsersAffected"},
|
|
|
139 "annotations": {
|
|
|
140 "summary": "net_routes is not getting regular updates"
|
|
|
141 },
|
|
|
142 },
|
|
|
143 ],
|
|
|
144 },
|
|
|
145 {
|
|
|
146 "name": "alerts",
|
|
|
147 "rules": [
|
|
|
148 {
|
|
|
149 "alert": "kube_node_status_bad_condition",
|
|
|
150 "for": "2h",
|
|
|
151 "labels": {"severity": "warning"},
|
|
|
152 "expr": 'kube_node_status_condition{condition=~".*Pressure",status="true"} > 0',
|
|
|
153 },
|
|
|
154 {
|
|
|
155 "alert": "housePower",
|
|
|
156 "for": "24h",
|
|
|
157 "labels": {"severity": "waste"},
|
|
|
158 "expr": "house_power_w > 4000",
|
|
|
159 "annotations": {"summary": "house power usage over 4KW"},
|
|
|
160 },
|
|
|
161 {
|
|
|
162 "alert": "host_root_fs_space_low",
|
|
|
163 "for": "20m",
|
|
|
164 "labels": {"severity": "warning"},
|
|
|
165 "expr": 'disk_free{path="/"} < 20G',
|
|
|
166 "annotations": {"summary": "low disk_free"},
|
|
|
167 },
|
|
|
168 {
|
|
|
169 "alert": "zpool_space_low",
|
|
|
170 "for": "20m",
|
|
|
171 "labels": {"severity": "warning"},
|
|
|
172 "expr": 'last_over_time(zfs_pool_free_bytes{pool="stor7"}[1h]) < 100G',
|
|
|
173 "annotations": {"summary": "low disk_free"},
|
|
|
174 },
|
|
|
175 {
|
|
|
176 "alert": "zpool_device_error_count",
|
|
|
177 "for": "20m",
|
|
|
178 "labels": {"severity": "warning"},
|
|
|
179 "expr": 'increase(zpool_device_error_count[1d]) > 0',
|
|
|
180 "annotations": {"summary": "low disk_free"},
|
|
|
181 },
|
|
|
182 {
|
|
|
183 "alert": "disk_week_incr",
|
|
|
184 "for": "20m",
|
|
|
185 "labels": {"severity": "warning"},
|
|
|
186 "expr": 'round(increase(disk_used{path=~"/my/.*"}[1d])/1M) > 5000',
|
|
|
187 "annotations": {"summary": "high mb/week on zfs dir"},
|
|
|
188 },
|
|
|
189 {
|
|
|
190 "alert": "high_logging",
|
|
|
191 "for": "20m",
|
|
|
192 "labels": {"severity": "waste"},
|
|
|
193 "expr": "sum by (container) (rate(kubelet_container_log_filesystem_used_bytes[3h])) > 4k",
|
|
|
194 "annotations": {"summary": "high log output rate"},
|
|
|
195 },
|
|
|
196 {
|
|
|
197 "alert": "stale_process",
|
|
|
198 "for": "1d",
|
|
|
199 "labels": {"severity": "dataRisk"},
|
|
|
200 "expr": "round((time() - filestat_modification_time/1e9) / 86400) > 14",
|
|
|
201 "annotations": {"summary": "process time is old"},
|
|
|
202 },
|
|
|
203 {
|
|
|
204 "alert": "starlette",
|
|
|
205 "for": "1m",
|
|
|
206 "labels": {"severity": "fix"},
|
|
|
207 "expr": 'starlette_request_duration_seconds_created{app_name="starlette"}',
|
|
|
208 "annotations": {"summary": "set starlette app name"},
|
|
|
209 },
|
|
|
210 {
|
|
|
211 "alert": "ssl_certs_expiring_soon",
|
|
|
212 "expr": "min((min_over_time(probe_ssl_earliest_cert_expiry[1d])-time())/86400) < 10",
|
|
|
213 "labels": {"severity": "warning"},
|
|
|
214 "annotations": {
|
|
|
215 "summary": "cert expiring soon. See https://bigasterisk.com/grafana/d/z1YtDa3Gz/certs?orgId=1\nVALUE = {{ $value }}"
|
|
|
216 },
|
|
|
217 },
|
|
|
218 ],
|
|
|
219 },
|
|
|
220 ]
|
|
|
221 }
|
|
|
222
|
|
|
223
|
|
|
224 def _runJson(ctx, cmd):
|
|
|
225 return json.loads(ctx.run(cmd, hide="stdout").stdout)
|
|
|
226
|
|
|
227
|
|
|
228 def hostsExpectedOnline(ctx):
|
|
|
229 return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py")
|
|
|
230
|
|
|
231
|
|
|
232 def expectedK8sNodes(ctx):
|
|
|
233 getNode = _runJson(ctx, "kubectl get node -o json")
|
|
|
234 hosts = [item["metadata"]["name"] for item in getNode["items"]]
|
|
|
235 optionalHosts = {'slash'}
|
|
|
236 return {
|
|
|
237 "groups": [
|
|
|
238 {
|
|
|
239 "name": "k8s_expected_nodes",
|
|
|
240 "rules": [
|
|
|
241 {
|
|
|
242 "alert": "kube_node_log_size_report_" + h,
|
|
|
243 "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})'
|
|
|
244 % h,
|
|
|
245 "for": "1h",
|
|
|
246 "annotations": {
|
|
|
247 "summary": f"no recent k8s log size report from host {h}"
|
|
|
248 },
|
|
|
249 }
|
|
|
250 for h in hosts if not h in optionalHosts
|
|
|
251 ],
|
|
|
252 }
|
|
|
253 ]
|
|
|
254 }
|
