Mercurial > code > home > repos > victoriametrics
changeset 60:e3151ab43579
moving scrape target configs in py, and into separate retention classes
| author | drewp@bigasterisk.com |
|---|---|
| date | Sat, 27 Apr 2024 01:31:07 -0700 |
| parents | 6a357b483c04 |
| children | fb0519859645 |
| files | config/create_scrape_configs.py config/scrape_forever.yaml next/create_all.py tasks.py |
| diffstat | 4 files changed, 105 insertions(+), 42 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/config/create_scrape_configs.py Sat Apr 27 01:31:07 2024 -0700 @@ -0,0 +1,82 @@ +import json +from pathlib import Path + +forever_jobs = [ + 'maildir-count', + 'mongodb', + 'net-traffic', + 'ping', + 'power-eagle', + 'smartctl', + 'wifi', + 'zfs-exporter', + 'zigbee2mqtt-exporter', + 'zpool-exporter', +] + +job_targets = { + 'maildir-count': ['prime:2500'], + 'mongodb': ['mongodb:9216'], + 'net-traffic': ['pipe:8080'], + # - job_name: "ping" + # scrape_interval: 2m + # metrics_path: /probe + # params: + # module: [icmp] + # static_configs: + # - targets: + # # printer, since it falls out of ntop with no traffic at all. Or, we could poll ink status at http://printer/general/status.html?pageid=1 + # - printer + # # wireguard connection test + # - prime5 + # # after pyinfra or reboot, seems to lose wg0 address + # - garage5 + # relabel_configs: + # - {source_labels: [__address__], target_label: __param_target} + # - {source_labels: [__param_target], target_label: instance} + # - target_label: __address__ + # replacement: prober + 'ping': [':80'], # multi target with tricks + 'power-eagle': ['power-eagle:80'], + 'smartctl': [':80'], # per-node! + 'wifi': ['wifi:80'], + 'zfs-exporter': ['ditto:9634'], + 'zigbee2mqtt-exporter': ['zigbee2mqtt-exporter:80'], + 'zpool-exporter': ['ditto:9986'], +} + +for job in forever_jobs: + d = [{ + "job_name": job, + "static_configs": [{ + "targets": job_targets[job], + }], + "relabel_configs": [ + { + "target_label": "namespace", + "replacement": "default" + }, + { + "source_labels": ["__meta_kubernetes_pod_node_name"], + "target_label": "node" + }, + ] + }] + + if job == 'power-eagle': + # from powerEagle/private_config.periodSec + d[0]['relabel_configs'].append({ + "action": "replace", + "target_label": "__scrape_interval__", + "replacement": "8s", + }) + + Path(f'{job}.yaml').write_text(json.dumps(d, indent=2, sort_keys=True)) + +Path('scrape_forever.yaml').write_text(json.dumps({ + "global": { + "scrape_interval": "1m", + "scrape_timeout": "10s" + }, + "scrape_config_files": [f'{job}.yaml' for job in forever_jobs], +}, indent=2))
--- a/config/scrape_forever.yaml Fri Apr 26 23:37:14 2024 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,38 +0,0 @@ -# see https://relabeler.promlabs.com/ - -global: - scrape_interval: 1m - scrape_timeout: 10s - -# scrape_config_files: -# - build/scrape_ssl.yaml -# These can even be urls: https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md#loading-scrape-configs-from-multiple-files - -scrape_configs: - # some based on https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus-kubernetes.yml - - - job_name: "k8services" - kubernetes_sd_configs: [{ role: endpoints }] - relabel_configs: - - # promote these to display - - source_labels: [__meta_kubernetes_service_name] - target_label: job - - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - - # for convenience in this config - - source_labels: [__meta_kubernetes_pod_container_port_number] - target_label: __port_number - - # period tweaks - - if: '{job="power-eagle"}' - action: replace - target_label: __scrape_interval__ - # from powerEagle/private_config.periodSec - replacement: 8s - - - if: '{job!="power-eagle"}' - action: drop -
--- a/next/create_all.py Fri Apr 26 23:37:14 2024 -0700 +++ b/next/create_all.py Sat Apr 27 01:31:07 2024 -0700 @@ -3,7 +3,7 @@ from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc -def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName): +def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix): (build / f'{agentFileName}_deploy.yaml').write_text( toJson({ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName }, @@ -15,7 +15,7 @@ "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" } }, "spec": { - "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }], + "volumes": [{ "name": "config", "configMap": { "name": f"{objPrefix}-config" } }], "serviceAccountName": "victoriametrics", "containers": [{ "name": "vmagent", @@ -154,7 +154,7 @@ request = "50Gi" pipelineWebRoot = f'{webRoot}/{pipelineName}' - createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName) + createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix) createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort) createPv(storageFileName, volName, request) createPvc(storageFileName, volName, request) @@ -221,7 +221,7 @@ ('forever', '100y'), ('recent', '90y'), ] - storageSvcs = [createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, p, f'scrape_{p}', ret) for p, ret in pipelines] + storageSvcs = [createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, p, f'scrape_{p}.yaml', ret) for p, ret in pipelines] selectPort = 8481 createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort)
--- a/tasks.py Fri Apr 26 23:37:14 2024 -0700 +++ b/tasks.py Sat Apr 27 01:31:07 2024 -0700 @@ -34,11 +34,30 @@ # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod') +@task +def push_config_2024(ctx): + # plan: + # every discovered service may: + # - be described here as a forever retention - ignore the discovery + # - be blocked here as a no-metrics service - ignore the discovery + # - be scraped as 'recent', with possible overrides of port/path + # all per-node metrics shall be 'recent' (oops, not smartctl!) + map: dict[str, object] = { + 'rules': alert_rules.allRules(ctx), + } + for p in Path('config').glob('*.yaml'): + map[p.name] = scrapeConfig(p) + replaceCmap("next-victoriametrics-config", map) + refreshPodCmaps(firstPodName("app=next-victoriametrics-forever-vmagent")) + + def scrapeConfig(fn): return yaml.load(open(fn), yaml.FullLoader) metricsToMigrate = [ + 'currently_on_wifi', + 'connected', 'house_power_kwh', 'house_power_price', 'house_power_w',