annotate config/scrape_main.yaml @ 64:def1aa2bfa3f

more targets polish. reorg code into next/
author drewp@bigasterisk.com
date Thu, 02 May 2024 23:06:31 -0700
parents 8199713c9294
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
4
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
1 # some based on https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus-kubernetes.yml
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
2
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
3 - job_name: "k8services"
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
4 kubernetes_sd_configs: [{ role: endpoints }]
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
5 relabel_configs:
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
6 # To omit a service, add this at pod-level (Deployment.spec.template.metadata.annotations):
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
7 # annotations: { prometheus.io/scrape: "false" }
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
8 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
9
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
9 regex: "false"
4
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
10 action: drop
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
11
19
10017def57ce update scrapes config
drewp@bigasterisk.com
parents: 17
diff changeset
12 # - source_labels: [__meta_kubernetes_namespace]
10017def57ce update scrapes config
drewp@bigasterisk.com
parents: 17
diff changeset
13 # regex: default
10017def57ce update scrapes config
drewp@bigasterisk.com
parents: 17
diff changeset
14 # action: keep
4
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
15
9
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
16 # promote these to display
4
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
17 - source_labels: [__meta_kubernetes_service_name]
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
18 target_label: job
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
19
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
20 - source_labels: [__meta_kubernetes_pod_node_name]
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
21 target_label: node
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
22
9
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
23 # for convenience in this config
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
24 - source_labels: [__meta_kubernetes_pod_container_port_number]
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
25 target_label: __port_number
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
26
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
27 # path tweaks
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
28 - if: '{job="victoriametrics",__port_number="8428"}'
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
29 action: replace
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
30 target_label: "__metrics_path__"
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
31 replacement: "/m/metrics"
29
a4c49fa01c9d correct v-logs path
drewp@bigasterisk.com
parents: 25
diff changeset
32 - if: '{job="victorialogs",__port_number="9428"}'
a4c49fa01c9d correct v-logs path
drewp@bigasterisk.com
parents: 25
diff changeset
33 action: replace
a4c49fa01c9d correct v-logs path
drewp@bigasterisk.com
parents: 25
diff changeset
34 target_label: "__metrics_path__"
a4c49fa01c9d correct v-logs path
drewp@bigasterisk.com
parents: 25
diff changeset
35 replacement: "/logs/metrics"
25
4bf507b38e34 video-files scrape cases
drewp@bigasterisk.com
parents: 19
diff changeset
36 - if: '{job="video-files",__port_number="8004"}'
4bf507b38e34 video-files scrape cases
drewp@bigasterisk.com
parents: 19
diff changeset
37 action: replace
4bf507b38e34 video-files scrape cases
drewp@bigasterisk.com
parents: 19
diff changeset
38 target_label: "__metrics_path__"
4bf507b38e34 video-files scrape cases
drewp@bigasterisk.com
parents: 19
diff changeset
39 replacement: "/video/api/metrics"
4
1eb6e6a2b9b6 version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff changeset
40
9
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
41 # discovery is matching extra ports that don't serve metrics- remove these targets
38
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
42 - {if: '{job="cert-manager-webhook"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
43 - {if: '{job="cert-manager", __port_number="9403"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
44 - {if: '{job="filesync-syncthing",__port_number="21027"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
45 - {if: '{job="filesync-syncthing",__port_number="22000"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
46 - {if: '{job="filesync-syncthing",__port_number="8384"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
47 - {if: '{job="jsregistry", __port_number="4873"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
48 - {if: '{job="kube-dns", __port_number="53"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
49 - {if: '{job="kubernetes"}', action: drop}
9
17db5e8e7a2f big rules and scrape config updates
drewp@bigasterisk.com
parents: 6
diff changeset
50 - {if: '{job="mongodb", __port_number="27017"}', action: drop}
38
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
51 - {if: '{job="mosquitto-ext", __port_number="1883"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
52 - {if: '{job="net-route-input", __port_number="80"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
53 - {if: '{job="photoprism", __port_number="2342"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
54 - {if: '{job="pomerium-metrics", __port_number="8080"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
55 - {if: '{job="pomerium-metrics", __port_number="8443"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
56 - {if: '{job="pomerium-proxy", __port_number="8080"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
57 - {if: '{job="pomerium-proxy", __port_number="8443"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
58 - {if: '{job="video-files", __port_number="8003"}', action: drop}
ddcaa1abe0c5 more ignorelist
drewp@bigasterisk.com
parents: 34
diff changeset
59 - {if: '{job=~"cm-acme-.*"}', action: drop}
50
16bde029b19f +filebeat, -some errors
drewp@bigasterisk.com
parents: 45
diff changeset
60 - {if: '{job="nvidiagpu-node-feature-discovery-master", __port_number="8080"}', action: drop}
41
407ee7fbda13 rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents: 39
diff changeset
61
407ee7fbda13 rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents: 39
diff changeset
62 # already have this with a job="pomerium-proxy"
407ee7fbda13 rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents: 39
diff changeset
63 - {if: '{job="pomerium-metrics"}', action: drop}
407ee7fbda13 rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents: 39
diff changeset
64
407ee7fbda13 rm double metrics; add alert for too-many-500s
drewp@bigasterisk.com
parents: 39
diff changeset
65
33
ce7ae4d2b24b just block all 8001/8002
drewp@bigasterisk.com
parents: 29
diff changeset
66
34
3b91d52b007d rules tuning
drewp@bigasterisk.com
parents: 33
diff changeset
67 # Assume all 8001/8002 port discoveries are redundant with an nginx proxy
33
ce7ae4d2b24b just block all 8001/8002
drewp@bigasterisk.com
parents: 29
diff changeset
68 - {if: '{__port_number="8001"}', action: drop}
ce7ae4d2b24b just block all 8001/8002
drewp@bigasterisk.com
parents: 29
diff changeset
69 - {if: '{__port_number="8002"}', action: drop}
19
10017def57ce update scrapes config
drewp@bigasterisk.com
parents: 17
diff changeset
70
10017def57ce update scrapes config
drewp@bigasterisk.com
parents: 17
diff changeset
71 # Needs https. Used by `kubectl top`
44
drewp@bigasterisk.com
parents: 42
diff changeset
72 - {if: '{job="metrics-server", __port_number="443"}', action: drop}
50
16bde029b19f +filebeat, -some errors
drewp@bigasterisk.com
parents: 45
diff changeset
73 # Something doesn't work with the scrape, and I don't see why I should care:
16bde029b19f +filebeat, -some errors
drewp@bigasterisk.com
parents: 45
diff changeset
74 - {if: '{job="metrics-server" }', action: drop}
16bde029b19f +filebeat, -some errors
drewp@bigasterisk.com
parents: 45
diff changeset
75