Mercurial > code > home > repos > victoriametrics
annotate config/scrape_main.yaml @ 52:8199713c9294
+ping garage5
author | drewp@bigasterisk.com |
---|---|
date | Wed, 24 Apr 2024 11:36:36 -0700 |
parents | 16bde029b19f |
children | def1aa2bfa3f |
rev | line source |
---|---|
9 | 1 # see https://relabeler.promlabs.com/ |
2 | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
3 global: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
4 scrape_interval: 1m |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
5 scrape_timeout: 10s |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
6 |
9 | 7 # scrape_config_files: |
8 # - build/scrape_ssl.yaml | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
9 # These can even be urls: https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md#loading-scrape-configs-from-multiple-files |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
10 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
11 scrape_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
12 # some based on https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus-kubernetes.yml |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
13 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
14 - job_name: "kubernetes-apiservers" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
15 scheme: https |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
16 tls_config: { ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt } |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
17 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
18 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
19 kubernetes_sd_configs: [{ role: endpoints }] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
20 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
21 relabel_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
22 - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
23 action: keep |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
24 regex: default;kubernetes;https |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
25 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
26 - job_name: "kubernetes-nodes" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
27 scheme: https |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
28 tls_config: { ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt } |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
29 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
30 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
31 kubernetes_sd_configs: [{ role: node }] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
32 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
33 relabel_configs: |
45
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
34 - action: labeldrop |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
35 regex: "__meta_kubernetes_node_label_(feature_node|nvidia_com_|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
36 - action: labelmap |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
37 regex: __meta_kubernetes_node_label_(.+) |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
38 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
39 # see https://github.com/google/cadvisor/blob/master/docs/storage/prometheus.md |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
40 # for metric definitions |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
41 - job_name: "kubernetes-cadvisor" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
42 scheme: https |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
43 metrics_path: /metrics/cadvisor |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
44 tls_config: { ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt } |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
45 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
46 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
47 kubernetes_sd_configs: [{ role: node }] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
48 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
49 relabel_configs: |
45
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
50 - action: labeldrop |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
51 regex: "(feature_node|nvidia_com_gpu|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
52 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
53 - job_name: "k8services" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
54 kubernetes_sd_configs: [{ role: endpoints }] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
55 relabel_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
56 # To omit a service, add this at pod-level (Deployment.spec.template.metadata.annotations): |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
57 # annotations: { prometheus.io/scrape: "false" } |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
58 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] |
9 | 59 regex: "false" |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
60 action: drop |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
61 |
19 | 62 # - source_labels: [__meta_kubernetes_namespace] |
63 # regex: default | |
64 # action: keep | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
65 |
9 | 66 # promote these to display |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
67 - source_labels: [__meta_kubernetes_service_name] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
68 target_label: job |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
69 |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
70 - source_labels: [__meta_kubernetes_pod_node_name] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
71 target_label: node |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
72 |
9 | 73 # for convenience in this config |
74 - source_labels: [__meta_kubernetes_pod_container_port_number] | |
75 target_label: __port_number | |
76 | |
77 # period tweaks | |
78 - if: '{job="power-eagle"}' | |
79 action: replace | |
80 target_label: __scrape_interval__ | |
81 # from powerEagle/private_config.periodSec | |
82 replacement: 8s | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
83 |
9 | 84 # path tweaks |
85 - if: '{job="victoriametrics",__port_number="8428"}' | |
86 action: replace | |
87 target_label: "__metrics_path__" | |
88 replacement: "/m/metrics" | |
29 | 89 - if: '{job="victorialogs",__port_number="9428"}' |
90 action: replace | |
91 target_label: "__metrics_path__" | |
92 replacement: "/logs/metrics" | |
25 | 93 - if: '{job="video-files",__port_number="8004"}' |
94 action: replace | |
95 target_label: "__metrics_path__" | |
96 replacement: "/video/api/metrics" | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
97 |
9 | 98 # discovery is matching extra ports that don't serve metrics- remove these targets |
38 | 99 - {if: '{job="cert-manager-webhook"}', action: drop} |
100 - {if: '{job="cert-manager", __port_number="9403"}', action: drop} | |
101 - {if: '{job="filesync-syncthing",__port_number="21027"}', action: drop} | |
102 - {if: '{job="filesync-syncthing",__port_number="22000"}', action: drop} | |
103 - {if: '{job="filesync-syncthing",__port_number="8384"}', action: drop} | |
104 - {if: '{job="jsregistry", __port_number="4873"}', action: drop} | |
105 - {if: '{job="kube-dns", __port_number="53"}', action: drop} | |
106 - {if: '{job="kubernetes"}', action: drop} | |
9 | 107 - {if: '{job="mongodb", __port_number="27017"}', action: drop} |
38 | 108 - {if: '{job="mosquitto-ext", __port_number="1883"}', action: drop} |
109 - {if: '{job="net-route-input", __port_number="80"}', action: drop} | |
110 - {if: '{job="photoprism", __port_number="2342"}', action: drop} | |
111 - {if: '{job="pomerium-metrics", __port_number="8080"}', action: drop} | |
112 - {if: '{job="pomerium-metrics", __port_number="8443"}', action: drop} | |
113 - {if: '{job="pomerium-proxy", __port_number="8080"}', action: drop} | |
114 - {if: '{job="pomerium-proxy", __port_number="8443"}', action: drop} | |
115 - {if: '{job="video-files", __port_number="8003"}', action: drop} | |
116 - {if: '{job=~"cm-acme-.*"}', action: drop} | |
50 | 117 - {if: '{job="nvidiagpu-node-feature-discovery-master", __port_number="8080"}', action: drop} |
41
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
39
diff
changeset
|
118 |
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
39
diff
changeset
|
119 # already have this with a job="pomerium-proxy" |
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
39
diff
changeset
|
120 - {if: '{job="pomerium-metrics"}', action: drop} |
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
39
diff
changeset
|
121 |
407ee7fbda13
rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents:
39
diff
changeset
|
122 |
33 | 123 |
34 | 124 # Assume all 8001/8002 port discoveries are redundant with an nginx proxy |
33 | 125 - {if: '{__port_number="8001"}', action: drop} |
126 - {if: '{__port_number="8002"}', action: drop} | |
19 | 127 |
128 # Needs https. Used by `kubectl top` | |
44 | 129 - {if: '{job="metrics-server", __port_number="443"}', action: drop} |
50 | 130 # Something doesn't work with the scrape, and I don't see why I should care: |
131 - {if: '{job="metrics-server" }', action: drop} | |
132 | |
17 | 133 |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
134 - job_name: "telegraf" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
135 scheme: http |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
136 kubernetes_sd_configs: [{ role: node }] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
137 relabel_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
138 - source_labels: [__address__] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
139 regex: "(.*):(\\d+)" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
140 target_label: __address__ |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
141 replacement: "${1}:9273" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
142 action: replace |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
143 |
17 | 144 - job_name: "smartctl" |
145 scrape_interval: 1h | |
146 scheme: http | |
147 kubernetes_sd_configs: [{ role: node }] | |
148 relabel_configs: | |
149 - source_labels: [__address__] | |
150 regex: "(.*):(\\d+)" | |
151 target_label: __address__ | |
152 replacement: "${1}:9633" | |
153 action: replace | |
154 | |
50 | 155 - job_name: "filebeat" |
156 scrape_interval: 1m | |
157 scheme: http | |
158 kubernetes_sd_configs: [{ role: node }] | |
159 relabel_configs: | |
160 - source_labels: [__address__] | |
161 regex: "(.*):(\\d+)" | |
162 target_label: __address__ | |
163 replacement: "${1}:5067" | |
164 action: replace | |
165 | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
166 - job_name: "net-routes" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
167 static_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
168 - targets: |
9 | 169 - pipe:9999 |
170 | |
171 - job_name: "net-traffic" | |
172 static_configs: | |
173 - targets: | |
174 - pipe:8080 | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
175 |
11 | 176 - job_name: "dnsmasq-log" |
177 static_configs: | |
178 - targets: | |
179 - pipe:9991 | |
180 | |
39 | 181 - job_name: "maildir-count" |
182 static_configs: | |
183 - targets: | |
184 - prime:2500 | |
185 | |
17 | 186 - job_name: "zfs" |
187 scrape_interval: 1h | |
188 static_configs: | |
189 - targets: | |
19 | 190 # running in in k8s, but as daemonset so it's not in SD above |
17 | 191 - ditto:9634 |
192 - ditto:9986 | |
11 | 193 |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
194 - job_name: "ping" |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
195 scrape_interval: 2m |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
196 metrics_path: /probe |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
197 params: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
198 module: [icmp] |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
199 static_configs: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
200 - targets: |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
201 # printer, since it falls out of ntop with no traffic at all. Or, we could poll ink status at http://10.2.0.37/general/status.html?pageid=1 |
42 | 202 - printer |
203 # wireguard connection test | |
204 - prime5 | |
52 | 205 # after pyinfra or reboot, seems to lose wg0 address |
206 - garage5 | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
207 relabel_configs: |
9 | 208 - {source_labels: [__address__], target_label: __param_target} |
209 - {source_labels: [__param_target], target_label: instance} | |
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
210 - target_label: __address__ |
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
211 replacement: prober |
19 | 212 |
12 | 213 - job_name: "racc" |
214 scrape_interval: 30s | |
215 static_configs: | |
45
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
216 - targets: [] |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
217 # - dash:5150 |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
218 # - dot:5150 |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
219 # - plus:5150 |
e1db51416e73
cleanup some silenced errs and the new featurenode labels
drewp@bigasterisk.com
parents:
44
diff
changeset
|
220 # - Kelsis-iMac:5150 |