Mercurial > code > home > repos > victoriametrics
annotate create_scrape_configs.py @ 79:bbe646f61cb0
announce doesn't serve normal metrics
author | drewp@bigasterisk.com |
---|---|
date | Wed, 26 Jun 2024 22:29:30 -0700 |
parents | 009527a145d0 |
children | 5a526531305f |
rev | line source |
---|---|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
1 from pathlib import Path |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
2 |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
3 from scrape_job import jobConfig, scrape_deployments, writeJobConfigs, FromName |
61 | 4 import private |
5 | |
6 # previously this used `kubernetes_sd_configs: [{ role: node }]` | |
7 all_hosts = [ | |
8 'dash', | |
9 'ditto', | |
10 # 'ws-printer', | |
11 #todo: | |
12 ] | |
13 | |
14 smartctl_hosts = [ | |
15 # ideally, all nodes with disks, but many turn off and on | |
16 'dash', | |
17 'ditto', | |
18 ] | |
19 | |
20 ping_hosts = [ | |
21 # printer, since it falls out of ntop with no traffic at all. Or, we could poll ink status at http://printer/general/status.html?pageid=1 | |
22 'printer', | |
23 # wireguard connection test | |
24 'prime5', | |
25 # after pyinfra or reboot, seems to lose wg0 address | |
26 'garage5', | |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
27 ] |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
28 |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
29 |
61 | 30 forever_jobs = [ |
31 jobConfig(name='maildir-count', targets=['prime:2500']), | |
32 jobConfig(name='mongodb', targets=['mongodb:9216']), | |
33 jobConfig(name='net-traffic', targets=['pipe:8080']), | |
34 jobConfig(name='ping', targets=ping_hosts, scrape_interval='2m', ping_job=True), | |
35 jobConfig(name='power-eagle', targets=['power-eagle:80'], scrape_interval='8s'), # from powerEagle/private_config.periodSec | |
36 jobConfig(name='powermeter-exporter', targets=['powermeter-exporter'], scrape_interval='10s'), | |
37 jobConfig(name='smartctl', targets=[f'{h}:9633' for h in smartctl_hosts]), | |
38 jobConfig(name='wifi', targets=['wifi:80']), | |
39 jobConfig(name='zfs-exporter', targets=['ditto:9634']), | |
40 jobConfig(name='zigbee2mqtt-exporter', targets=['zigbee2mqtt-exporter:80']), | |
41 jobConfig(name='zpool-exporter', targets=['ditto:9986']), | |
42 jobConfig(name='octoprint', targets=['octoprint'], | |
43 metrics_path='/plugin/prometheus_exporter/metrics', | |
44 params={'apikey' : [private.octoprint_apikey]}, | |
45 ), | |
46 ] # yapf: disable | |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
47 |
61 | 48 recent_jobs = [ |
76
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
49 jobConfig(name="dnsmasq-log", targets=['pipe:9991']), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
50 jobConfig(name="filebeat", targets=[f'{h}:5067' for h in all_hosts]), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
51 jobConfig(name="kube-state-metrics", targets=['kube-state-metrics.kube-system:8080']), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
52 jobConfig(name="net-routes", targets=['pipe:9999']), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
53 jobConfig(name="net-traffic", targets=['pipe:8080']), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
54 jobConfig(name="pomerium", targets=['pomerium-metrics.pomerium:9090']), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
55 jobConfig(name="telegraf", targets=[f'{h}:9273' for h in all_hosts]), |
009527a145d0
add kube-state-metrics scrape; loosen some high-logging thresholds
drewp@bigasterisk.com
parents:
75
diff
changeset
|
56 jobConfig(name="victorialogs", targets=['victorialogs'], metrics_path='/logs/metrics'), |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
57 |
68 | 58 jobConfig(name="victoriametrics-forever-vmagent", metrics_path='/m/forever/vmagent/metrics', targets=FromName), |
59 jobConfig(name="victoriametrics-forever-vminsert", metrics_path='/m/forever/vminsert/metrics', targets=FromName), | |
60 jobConfig(name="victoriametrics-forever-vmstorage", metrics_path='/m/forever/vmstorage/metrics',targets=FromName), | |
61 jobConfig(name="victoriametrics-recent-vmagent", metrics_path='/m/recent/vmagent/metrics', targets=FromName), | |
62 jobConfig(name="victoriametrics-recent-vminsert", metrics_path='/m/recent/vminsert/metrics', targets=FromName), | |
63 jobConfig(name="victoriametrics-recent-vmstorage", metrics_path='/m/recent/vmstorage/metrics', targets=FromName), | |
64 jobConfig(name="victoriametrics-vmselect", metrics_path='/m/vmselect/metrics', targets=FromName), | |
72 | 65 jobConfig(name="victoriametrics-vmalert", metrics_path='/m/vmalert/metrics', targets=FromName), |
73 | 66 jobConfig(name="victoriametrics-alertmanager", metrics_path='/m/alertmanager/metrics', targets=FromName), |
68 | 67 jobConfig(name="victoriametrics-index", targets=FromName), |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
68 |
65 | 69 # todo: |
70 # - video-files | |
71 # - cert-manager | |
72 # - syncthing(s) | |
73 # - nvidia runner | |
75
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
74 # - longhorn (svc longhorn-backend, port 9500?) |
65 | 75 # - kube-system.metrics-server |
61 | 76 jobConfig( |
77 name="racc", | |
78 scrape_interval='30s', | |
79 targets=[ | |
80 # - dash:5150 | |
81 # - dot:5150 | |
82 # - squib:5150 | |
83 # - ashermac:5150 | |
84 ], | |
85 ), | |
86 ] # yapf: disable | |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
87 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
88 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
89 deploy_doesnt_serve_metrics = [ |
79 | 90 'announce', # serves them, but it's on an external ip, etc. |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
91 'apprise', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
92 'bitwarden', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
93 'digi-files', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
94 'digi-pose-predict', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
95 'digi-tts-mimic', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
96 'digi-web', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
97 'dovecot', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
98 'ectoscope', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
99 'front-door-display', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
100 'hass', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
101 'homepage', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
102 'itch150', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
103 'jsregistry', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
104 'kallithea', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
105 'kube-web-view', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
106 'magma', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
107 'megasecond', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
108 'minecraft-build-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
109 'minecraft-lake-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
110 'minecraft-smp-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
111 'mongodb', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
112 'mqtt1', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
113 'mqtt2', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
114 'nodered', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
115 'photoprism', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
116 'plik', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
117 'projects', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
118 'registry-ui', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
119 'registry', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
120 'speakerphone', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
121 'victorialogs-ui', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
122 'video-files', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
123 'video', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
124 'zigbee2mqtt', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
125 'zwave2mqtt', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
126 ] |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
127 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
128 existing_jobs = [j['job_name'] for j in forever_jobs + recent_jobs] |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
129 recent_jobs.extend(scrape_deployments(deploy_doesnt_serve_metrics + existing_jobs)) |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
130 |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
131 recent_jobs.append(jobConfig(name='kubernetes-apiservers', https=True, targets=[]) | { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
132 'kubernetes_sd_configs': [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
133 'role': 'endpoints' |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
134 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
135 'relabel_configs': [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
136 'source_labels': ['__meta_kubernetes_namespace', '__meta_kubernetes_service_name', '__meta_kubernetes_endpoint_port_name'], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
137 'action': 'keep', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
138 'regex': 'default;kubernetes;https' |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
139 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
140 }) |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
141 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
142 recent_jobs.append( |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
143 jobConfig(name="kubernetes-nodes", https=True, targets=[]) | { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
144 "kubernetes_sd_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
145 "role": "node" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
146 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
147 "relabel_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
148 "action": "labeldrop", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
149 "regex": "__meta_kubernetes_node_label_(feature_node|nvidia_com_|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
150 }, { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
151 "action": "labelmap", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
152 "regex": "__meta_kubernetes_node_label_(.+)" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
153 }, { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
154 "action": "labeldrop", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
155 "regex": "kubernetes_io_hostname" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
156 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
157 }) |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
158 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
159 # see https://github.com/google/cadvisor/blob/master/docs/storage/prometheus.md |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
160 # for metric definitions |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
161 |
75
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
162 recent_jobs.append( |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
163 jobConfig(name="kubernetes-cadvisor", https=True, metrics_path="/metrics/cadvisor", targets=[]) | { |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
164 "kubernetes_sd_configs": [{ |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
165 "role": "node" |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
166 }], |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
167 "relabel_configs": [{ |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
168 "action": "labeldrop", |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
169 "regex": "(feature_node|nvidia_com_gpu|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
170 }, { |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
171 "action": "replace", |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
172 "target_label": "__metrics_path__", |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
173 "replacement": '/metrics/cadvisor', |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
174 }], |
37fb428d5013
workaround for cadvisor metrics path not getting set
drewp@bigasterisk.com
parents:
73
diff
changeset
|
175 }) |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
176 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
177 outDir = Path('build/scrape_config') |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
178 writeJobConfigs(outDir, forever_jobs, 'forever') |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
179 writeJobConfigs(outDir, recent_jobs, 'recent') |