Mercurial > code > home > repos > victoriametrics
annotate tasks.py @ 61:fb0519859645
better py configs
| author | drewp@bigasterisk.com |
|---|---|
| date | Thu, 02 May 2024 18:35:46 -0700 |
| parents | e3151ab43579 |
| children | 8134cd480817 |
| rev | line source |
|---|---|
| 59 | 1 from pathlib import Path |
| 20 | 2 import yaml |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
3 from invoke import task |
| 20 | 4 from kubernetes import config |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
5 |
|
22
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
6 import alert_rules |
| 27 | 7 from k8s_ops import firstPodName, refreshPodCmaps, replaceCmap |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
8 |
| 20 | 9 config.load_kube_config() |
| 12 | 10 |
| 11 | |
|
22
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
12 @task |
|
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
13 def push_config(ctx): |
|
32
eb1de82c93aa
refactor the merging of all the groups
drewp@bigasterisk.com
parents:
27
diff
changeset
|
14 rulesObj = alert_rules.allRules(ctx) |
| 10 | 15 |
| 54 | 16 replaceCmap("victoriametrics-config", { |
| 17 "scrape_main": scrapeConfig("config/scrape_main.yaml"), | |
|
58
0064c490c33d
some tiny scrape configs for testing multi-node vmstorage/vmselect
drewp@bigasterisk.com
parents:
54
diff
changeset
|
18 "scrape_recent": scrapeConfig("config/scrape_recent.yaml"), |
|
0064c490c33d
some tiny scrape configs for testing multi-node vmstorage/vmselect
drewp@bigasterisk.com
parents:
54
diff
changeset
|
19 "scrape_forever": scrapeConfig("config/scrape_forever.yaml"), |
| 54 | 20 "rules": rulesObj, |
| 21 }) | |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
22 |
| 34 | 23 # these don't give errors on rules format! they just quietly keep the old |
| 24 # rules! use `skaffold run` to get errs. | |
| 25 # | |
| 26 # or run | |
| 27 # validateTemplates = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates") | |
| 54 | 28 # validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine") |
| 34 | 29 |
|
22
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
30 refreshPodCmaps(firstPodName("app=victoriametrics")) |
|
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
31 refreshPodCmaps(firstPodName("app=vmalert")) |
|
4
1eb6e6a2b9b6
version control configs finally; use configmaps to present them to VM
drewp@bigasterisk.com
parents:
diff
changeset
|
32 |
|
22
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
33 # If the VM reloader isn't fast enough, we could do this too: |
|
cd115f1ca2a8
use configmaps and a special pod refresh trick
drewp@bigasterisk.com
parents:
20
diff
changeset
|
34 # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod') |
| 20 | 35 |
| 36 | |
|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
37 @task |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
38 def push_config_2024(ctx): |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
39 # plan: |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
40 # every discovered service may: |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
41 # - be described here as a forever retention - ignore the discovery |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
42 # - be blocked here as a no-metrics service - ignore the discovery |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
43 # - be scraped as 'recent', with possible overrides of port/path |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
44 # all per-node metrics shall be 'recent' (oops, not smartctl!) |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
45 map: dict[str, object] = { |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
46 'rules': alert_rules.allRules(ctx), |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
47 } |
| 61 | 48 top=Path('config/build/scrape_jobs') |
| 49 for p in top.glob('**/*.yaml'): | |
| 50 map[str(p.relative_to(top))] = scrapeConfig(p) | |
|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
51 replaceCmap("next-victoriametrics-config", map) |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
52 refreshPodCmaps(firstPodName("app=next-victoriametrics-forever-vmagent")) |
| 61 | 53 refreshPodCmaps(firstPodName("app=next-victoriametrics-recent-vmagent")) |
|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
54 |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
55 |
| 54 | 56 def scrapeConfig(fn): |
| 57 return yaml.load(open(fn), yaml.FullLoader) | |
| 59 | 58 |
| 59 | |
| 60 metricsToMigrate = [ | |
|
60
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
61 'currently_on_wifi', |
|
e3151ab43579
moving scrape target configs in py, and into separate retention classes
drewp@bigasterisk.com
parents:
59
diff
changeset
|
62 'connected', |
| 59 | 63 'house_power_kwh', |
| 64 'house_power_price', | |
| 65 'house_power_w', | |
| 66 'lan_bytes_sent_from_created', | |
| 67 'lan_bytes_sent_from_total', | |
| 68 'lan_last_seen_time', | |
| 69 'maildir_count', | |
| 70 'mongodb_mongod_db_coll_avgobjsize', | |
| 71 'mongodb_mongod_db_coll_count', | |
| 72 'mongodb_mongod_db_coll_index_size', | |
| 73 'mongodb_mongod_db_coll_indexes', | |
| 74 'mongodb_mongod_db_coll_indexes_size', | |
| 75 'mongodb_mongod_db_coll_size', | |
| 76 'mongodb_mongod_db_coll_storage_size', | |
| 77 'mongodb_mongod_db_collections_total', | |
| 78 'mongodb_mongod_db_data_size_bytes', | |
| 79 'mongodb_mongod_db_index_size_bytes', | |
| 80 'mongodb_mongod_db_indexes_total', | |
| 81 'mongodb_mongod_db_objects_total', | |
| 82 'probe_duration_seconds', | |
| 83 'smartctl_device', | |
| 84 'smartctl_device_attribute', | |
| 85 'smartctl_device_available_spare', | |
| 86 'smartctl_device_available_spare_threshold', | |
| 87 'smartctl_device_block_size', | |
| 88 'smartctl_device_bytes_read', | |
| 89 'smartctl_device_bytes_written', | |
| 90 'smartctl_device_capacity_blocks', | |
| 91 'smartctl_device_capacity_bytes', | |
| 92 'smartctl_device_critical_warning', | |
| 93 'smartctl_device_interface_speed', | |
| 94 'smartctl_device_media_errors', | |
| 95 'smartctl_device_num_err_log_entries', | |
| 96 'smartctl_device_percentage_used', | |
| 97 'smartctl_device_power_cycle_count', | |
| 98 'smartctl_device_power_on_seconds', | |
| 99 'smartctl_device_smart_status', | |
| 100 'smartctl_device_smartctl_exit_status', | |
| 101 'smartctl_device_status', | |
| 102 'smartctl_device_temperature', | |
| 103 'zfs_dataset_logical_used_bytes', | |
| 104 'zfs_dataset_referenced_bytes', | |
| 105 'zfs_dataset_used_by_dataset_bytes', | |
| 106 'zfs_dataset_used_bytes', | |
| 107 'zfs_dataset_written_bytes', | |
| 108 'zfs_exporter_build_info', | |
| 109 'zfs_pool_allocated_bytes', | |
| 110 'zfs_pool_deduplication_ratio', | |
| 111 'zfs_pool_fragmentation_ratio', | |
| 112 'zfs_pool_free_bytes', | |
| 113 'zfs_pool_freeing_bytes', | |
| 114 'zfs_pool_health', | |
| 115 'zfs_pool_leaked_bytes', | |
| 116 'zfs_pool_readonly', | |
| 117 'zfs_pool_size_bytes', | |
| 118 'zfs_scrape_collector_duration_seconds', | |
| 119 'zfs_scrape_collector_success', | |
| 120 'zigbee_availability', | |
| 121 'zigbee_battery', | |
| 122 'zigbee_brightness', | |
| 123 'zigbee_contact', | |
| 124 'zigbee_link_quality', | |
| 125 'zigbee_occupancy', | |
| 126 'zpool_device_error_count', | |
| 127 'zpool_device_state', | |
| 128 'zpool_error_count', | |
| 129 'zpool_state', | |
| 130 ] | |
| 131 | |
| 132 | |
| 133 @task | |
| 134 def exportForeverMetrics(ctx): | |
| 135 svc = ctx.run('khost victoriametrics').stdout | |
| 136 for m in metricsToMigrate: | |
| 137 ctx.run(f'curl http://{svc}/m/api/v1/export/native?match={m} | gzip -c > export/{m}.native.gz') | |
| 138 | |
| 139 | |
| 140 @task | |
| 141 def ingestForeverMetrics(ctx): | |
| 142 svc = ctx.run('khost next-victoriametrics-forever-vminsert').stdout | |
| 143 for p in Path('export').glob('*.native.gz'): | |
| 144 print(f'importing {p}') | |
| 145 ctx.run(f'zcat {p} | curl -s http://{svc}/m/next/forever/vminsert/insert/0/prometheus/api/v1/import/native --data-binary @-') |