Mercurial > code > home > repos > victoriametrics
annotate create_scrape_configs.py @ 73:c3d13fab248e
alertmanager deployment
author | drewp@bigasterisk.com |
---|---|
date | Fri, 03 May 2024 14:40:29 -0700 |
parents | 85d9dae18656 |
children | 37fb428d5013 |
rev | line source |
---|---|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
1 from pathlib import Path |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
2 |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
3 from scrape_job import jobConfig, scrape_deployments, writeJobConfigs, FromName |
61 | 4 import private |
5 | |
6 # previously this used `kubernetes_sd_configs: [{ role: node }]` | |
7 all_hosts = [ | |
8 'dash', | |
9 'ditto', | |
10 # 'ws-printer', | |
11 #todo: | |
12 ] | |
13 | |
14 smartctl_hosts = [ | |
15 # ideally, all nodes with disks, but many turn off and on | |
16 'dash', | |
17 'ditto', | |
18 ] | |
19 | |
20 ping_hosts = [ | |
21 # printer, since it falls out of ntop with no traffic at all. Or, we could poll ink status at http://printer/general/status.html?pageid=1 | |
22 'printer', | |
23 # wireguard connection test | |
24 'prime5', | |
25 # after pyinfra or reboot, seems to lose wg0 address | |
26 'garage5', | |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
27 ] |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
28 |
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
29 |
61 | 30 forever_jobs = [ |
31 jobConfig(name='maildir-count', targets=['prime:2500']), | |
32 jobConfig(name='mongodb', targets=['mongodb:9216']), | |
33 jobConfig(name='net-traffic', targets=['pipe:8080']), | |
34 jobConfig(name='ping', targets=ping_hosts, scrape_interval='2m', ping_job=True), | |
35 jobConfig(name='power-eagle', targets=['power-eagle:80'], scrape_interval='8s'), # from powerEagle/private_config.periodSec | |
36 jobConfig(name='powermeter-exporter', targets=['powermeter-exporter'], scrape_interval='10s'), | |
37 jobConfig(name='smartctl', targets=[f'{h}:9633' for h in smartctl_hosts]), | |
38 jobConfig(name='wifi', targets=['wifi:80']), | |
39 jobConfig(name='zfs-exporter', targets=['ditto:9634']), | |
40 jobConfig(name='zigbee2mqtt-exporter', targets=['zigbee2mqtt-exporter:80']), | |
41 jobConfig(name='zpool-exporter', targets=['ditto:9986']), | |
42 jobConfig(name='octoprint', targets=['octoprint'], | |
43 metrics_path='/plugin/prometheus_exporter/metrics', | |
44 params={'apikey' : [private.octoprint_apikey]}, | |
45 ), | |
46 ] # yapf: disable | |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
47 |
61 | 48 recent_jobs = [ |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
49 jobConfig(name="dnsmasq-log", targets=['pipe:9991']), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
50 jobConfig(name="filebeat", targets=[f'{h}:5067' for h in all_hosts]), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
51 jobConfig(name="net-routes", targets=['pipe:9999']), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
52 jobConfig(name="net-traffic", targets=['pipe:8080']), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
53 jobConfig(name="pomerium", targets=['pomerium-metrics.pomerium:9090']), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
54 jobConfig(name="telegraf", targets=[f'{h}:9273' for h in all_hosts]), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
55 jobConfig(name="victorialogs",targets=['victorialogs'], metrics_path='/logs/metrics'), |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
56 |
68 | 57 jobConfig(name="victoriametrics-forever-vmagent", metrics_path='/m/forever/vmagent/metrics', targets=FromName), |
58 jobConfig(name="victoriametrics-forever-vminsert", metrics_path='/m/forever/vminsert/metrics', targets=FromName), | |
59 jobConfig(name="victoriametrics-forever-vmstorage", metrics_path='/m/forever/vmstorage/metrics',targets=FromName), | |
60 jobConfig(name="victoriametrics-recent-vmagent", metrics_path='/m/recent/vmagent/metrics', targets=FromName), | |
61 jobConfig(name="victoriametrics-recent-vminsert", metrics_path='/m/recent/vminsert/metrics', targets=FromName), | |
62 jobConfig(name="victoriametrics-recent-vmstorage", metrics_path='/m/recent/vmstorage/metrics', targets=FromName), | |
63 jobConfig(name="victoriametrics-vmselect", metrics_path='/m/vmselect/metrics', targets=FromName), | |
72 | 64 jobConfig(name="victoriametrics-vmalert", metrics_path='/m/vmalert/metrics', targets=FromName), |
73 | 65 jobConfig(name="victoriametrics-alertmanager", metrics_path='/m/alertmanager/metrics', targets=FromName), |
68 | 66 jobConfig(name="victoriametrics-index", targets=FromName), |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
67 |
65 | 68 # todo: |
69 # - video-files | |
70 # - cert-manager | |
71 # - syncthing(s) | |
72 # - nvidia runner | |
73 # - longhorn | |
74 # - kube-system.metrics-server | |
61 | 75 jobConfig( |
76 name="racc", | |
77 scrape_interval='30s', | |
78 targets=[ | |
79 # - dash:5150 | |
80 # - dot:5150 | |
81 # - squib:5150 | |
82 # - ashermac:5150 | |
83 ], | |
84 ), | |
85 ] # yapf: disable | |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
86 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
87 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
88 deploy_doesnt_serve_metrics = [ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
89 'apprise', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
90 'bitwarden', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
91 'digi-files', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
92 'digi-pose-predict', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
93 'digi-tts-mimic', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
94 'digi-web', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
95 'dovecot', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
96 'ectoscope', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
97 'front-door-display', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
98 'hass', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
99 'homepage', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
100 'itch150', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
101 'jsregistry', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
102 'kallithea', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
103 'kube-web-view', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
104 'magma', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
105 'megasecond', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
106 'minecraft-build-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
107 'minecraft-lake-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
108 'minecraft-smp-world', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
109 'mongodb', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
110 'mqtt1', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
111 'mqtt2', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
112 'nodered', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
113 'photoprism', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
114 'plik', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
115 'projects', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
116 'registry-ui', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
117 'registry', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
118 'speakerphone', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
119 'victorialogs-ui', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
120 'video-files', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
121 'video', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
122 'zigbee2mqtt', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
123 'zwave2mqtt', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
124 ] |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
125 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
126 existing_jobs = [j['job_name'] for j in forever_jobs + recent_jobs] |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
127 recent_jobs.extend(scrape_deployments(deploy_doesnt_serve_metrics + existing_jobs)) |
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
diff
changeset
|
128 |
64
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
129 recent_jobs.append(jobConfig(name='kubernetes-apiservers', https=True, targets=[]) | { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
130 'kubernetes_sd_configs': [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
131 'role': 'endpoints' |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
132 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
133 'relabel_configs': [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
134 'source_labels': ['__meta_kubernetes_namespace', '__meta_kubernetes_service_name', '__meta_kubernetes_endpoint_port_name'], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
135 'action': 'keep', |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
136 'regex': 'default;kubernetes;https' |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
137 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
138 }) |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
139 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
140 recent_jobs.append( |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
141 jobConfig(name="kubernetes-nodes", https=True, targets=[]) | { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
142 "kubernetes_sd_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
143 "role": "node" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
144 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
145 "relabel_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
146 "action": "labeldrop", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
147 "regex": "__meta_kubernetes_node_label_(feature_node|nvidia_com_|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
148 }, { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
149 "action": "labelmap", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
150 "regex": "__meta_kubernetes_node_label_(.+)" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
151 }, { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
152 "action": "labeldrop", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
153 "regex": "kubernetes_io_hostname" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
154 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
155 }) |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
156 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
157 # see https://github.com/google/cadvisor/blob/master/docs/storage/prometheus.md |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
158 # for metric definitions |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
159 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
160 recent_jobs.append(jobConfig(name="kubernetes-cadvisor", https=True, metrics_path="/metrics/cadvisor", targets=[]) | { |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
161 "kubernetes_sd_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
162 "role": "node" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
163 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
164 "relabel_configs": [{ |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
165 "action": "labeldrop", |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
166 "regex": "(feature_node|nvidia_com_gpu|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*" |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
167 }], |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
168 }) |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
169 |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
170 outDir = Path('build/scrape_config') |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
171 writeJobConfigs(outDir, forever_jobs, 'forever') |
def1aa2bfa3f
more targets polish. reorg code into next/
drewp@bigasterisk.com
parents:
62
diff
changeset
|
172 writeJobConfigs(outDir, recent_jobs, 'recent') |