Mercurial > code > home > repos > victoriametrics
changeset 63:84a4c4cca4a5
separate the migration tasks
| author | drewp@bigasterisk.com |
|---|---|
| date | Thu, 02 May 2024 23:05:14 -0700 |
| parents | 8134cd480817 |
| children | def1aa2bfa3f |
| files | migration/tasks.py tasks.py |
| diffstat | 2 files changed, 91 insertions(+), 87 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/migration/tasks.py Thu May 02 23:05:14 2024 -0700 @@ -0,0 +1,91 @@ +from pathlib import Path + +from invoke import task + + +metricsToMigrate = [ + 'currently_on_wifi', + 'connected', + 'house_power_kwh', + 'house_power_price', + 'house_power_w', + 'lan_bytes_sent_from_created', + 'lan_bytes_sent_from_total', + 'lan_last_seen_time', + 'maildir_count', + 'mongodb_mongod_db_coll_avgobjsize', + 'mongodb_mongod_db_coll_count', + 'mongodb_mongod_db_coll_index_size', + 'mongodb_mongod_db_coll_indexes', + 'mongodb_mongod_db_coll_indexes_size', + 'mongodb_mongod_db_coll_size', + 'mongodb_mongod_db_coll_storage_size', + 'mongodb_mongod_db_collections_total', + 'mongodb_mongod_db_data_size_bytes', + 'mongodb_mongod_db_index_size_bytes', + 'mongodb_mongod_db_indexes_total', + 'mongodb_mongod_db_objects_total', + 'probe_duration_seconds', + 'smartctl_device', + 'smartctl_device_attribute', + 'smartctl_device_available_spare', + 'smartctl_device_available_spare_threshold', + 'smartctl_device_block_size', + 'smartctl_device_bytes_read', + 'smartctl_device_bytes_written', + 'smartctl_device_capacity_blocks', + 'smartctl_device_capacity_bytes', + 'smartctl_device_critical_warning', + 'smartctl_device_interface_speed', + 'smartctl_device_media_errors', + 'smartctl_device_num_err_log_entries', + 'smartctl_device_percentage_used', + 'smartctl_device_power_cycle_count', + 'smartctl_device_power_on_seconds', + 'smartctl_device_smart_status', + 'smartctl_device_smartctl_exit_status', + 'smartctl_device_status', + 'smartctl_device_temperature', + 'zfs_dataset_logical_used_bytes', + 'zfs_dataset_referenced_bytes', + 'zfs_dataset_used_by_dataset_bytes', + 'zfs_dataset_used_bytes', + 'zfs_dataset_written_bytes', + 'zfs_exporter_build_info', + 'zfs_pool_allocated_bytes', + 'zfs_pool_deduplication_ratio', + 'zfs_pool_fragmentation_ratio', + 'zfs_pool_free_bytes', + 'zfs_pool_freeing_bytes', + 'zfs_pool_health', + 'zfs_pool_leaked_bytes', + 'zfs_pool_readonly', + 'zfs_pool_size_bytes', + 'zfs_scrape_collector_duration_seconds', + 'zfs_scrape_collector_success', + 'zigbee_availability', + 'zigbee_battery', + 'zigbee_brightness', + 'zigbee_contact', + 'zigbee_link_quality', + 'zigbee_occupancy', + 'zpool_device_error_count', + 'zpool_device_state', + 'zpool_error_count', + 'zpool_state', +] + + +@task +def exportForeverMetrics(ctx): + svc = ctx.run('khost victoriametrics').stdout + for m in metricsToMigrate: + ctx.run(f'curl http://{svc}/m/api/v1/export/native?match={m} | gzip -c > export/{m}.native.gz') + + +@task +def ingestForeverMetrics(ctx): + svc = ctx.run('khost next-victoriametrics-forever-vminsert').stdout + for p in Path('export').glob('*.native.gz'): + print(f'importing {p}') + ctx.run(f'zcat {p} | curl -s http://{svc}/m/next/forever/vminsert/insert/0/prometheus/api/v1/import/native --data-binary @-')
--- a/tasks.py Thu May 02 20:33:29 2024 -0700 +++ b/tasks.py Thu May 02 23:05:14 2024 -0700 @@ -57,90 +57,3 @@ def scrapeConfig(fn): return yaml.load(open(fn), yaml.FullLoader) - -metricsToMigrate = [ - 'currently_on_wifi', - 'connected', - 'house_power_kwh', - 'house_power_price', - 'house_power_w', - 'lan_bytes_sent_from_created', - 'lan_bytes_sent_from_total', - 'lan_last_seen_time', - 'maildir_count', - 'mongodb_mongod_db_coll_avgobjsize', - 'mongodb_mongod_db_coll_count', - 'mongodb_mongod_db_coll_index_size', - 'mongodb_mongod_db_coll_indexes', - 'mongodb_mongod_db_coll_indexes_size', - 'mongodb_mongod_db_coll_size', - 'mongodb_mongod_db_coll_storage_size', - 'mongodb_mongod_db_collections_total', - 'mongodb_mongod_db_data_size_bytes', - 'mongodb_mongod_db_index_size_bytes', - 'mongodb_mongod_db_indexes_total', - 'mongodb_mongod_db_objects_total', - 'probe_duration_seconds', - 'smartctl_device', - 'smartctl_device_attribute', - 'smartctl_device_available_spare', - 'smartctl_device_available_spare_threshold', - 'smartctl_device_block_size', - 'smartctl_device_bytes_read', - 'smartctl_device_bytes_written', - 'smartctl_device_capacity_blocks', - 'smartctl_device_capacity_bytes', - 'smartctl_device_critical_warning', - 'smartctl_device_interface_speed', - 'smartctl_device_media_errors', - 'smartctl_device_num_err_log_entries', - 'smartctl_device_percentage_used', - 'smartctl_device_power_cycle_count', - 'smartctl_device_power_on_seconds', - 'smartctl_device_smart_status', - 'smartctl_device_smartctl_exit_status', - 'smartctl_device_status', - 'smartctl_device_temperature', - 'zfs_dataset_logical_used_bytes', - 'zfs_dataset_referenced_bytes', - 'zfs_dataset_used_by_dataset_bytes', - 'zfs_dataset_used_bytes', - 'zfs_dataset_written_bytes', - 'zfs_exporter_build_info', - 'zfs_pool_allocated_bytes', - 'zfs_pool_deduplication_ratio', - 'zfs_pool_fragmentation_ratio', - 'zfs_pool_free_bytes', - 'zfs_pool_freeing_bytes', - 'zfs_pool_health', - 'zfs_pool_leaked_bytes', - 'zfs_pool_readonly', - 'zfs_pool_size_bytes', - 'zfs_scrape_collector_duration_seconds', - 'zfs_scrape_collector_success', - 'zigbee_availability', - 'zigbee_battery', - 'zigbee_brightness', - 'zigbee_contact', - 'zigbee_link_quality', - 'zigbee_occupancy', - 'zpool_device_error_count', - 'zpool_device_state', - 'zpool_error_count', - 'zpool_state', -] - - -@task -def exportForeverMetrics(ctx): - svc = ctx.run('khost victoriametrics').stdout - for m in metricsToMigrate: - ctx.run(f'curl http://{svc}/m/api/v1/export/native?match={m} | gzip -c > export/{m}.native.gz') - - -@task -def ingestForeverMetrics(ctx): - svc = ctx.run('khost next-victoriametrics-forever-vminsert').stdout - for p in Path('export').glob('*.native.gz'): - print(f'importing {p}') - ctx.run(f'zcat {p} | curl -s http://{svc}/m/next/forever/vminsert/insert/0/prometheus/api/v1/import/native --data-binary @-')