--- a/next/create_scrape_configs.py	Thu May 02 23:05:14 2024 -0700
+++ b/next/create_scrape_configs.py	Thu May 02 23:06:31 2024 -0700
@@ -1,11 +1,8 @@
 from pathlib import Path
 
-from scrape_job import jobConfig, scrape_deployments, writeJobConfigs
+from scrape_job import jobConfig, scrape_deployments, writeJobConfigs, FromName
 import private
 
-
-
-
 # previously this used `kubernetes_sd_configs: [{ role: node }]`
 all_hosts = [
     'dash',
@@ -29,38 +26,6 @@
     'garage5',
 ]
 
-deploy_doesnt_serve_metrics = [
-    'apprise',
-    'bitwarden',
-    'digi-files',
-    'digi-pose-predict',
-    'digi-tts-mimic',
-    'dovecot',
-    'front-door-display',
-    'hass',
-    'homepage',
-    'itch150',
-    'kallithea',
-    'kube-web-view',
-    'magma',
-    'megasecond',
-    'minecraft-build-world',
-    'minecraft-lake-world',
-    'minecraft-smp-world',
-    'mongodb',
-    'mqtt1',
-    'mqtt2',
-    'nodered',
-    'photoprism',
-    'plik',
-    'projects',
-    'registry',
-    'registry-ui',
-    'speakerphone',
-    'video',
-    'video-files',
-    'zigbee2mqtt',
-]
 
 forever_jobs = [
     jobConfig(name='maildir-count',        targets=['prime:2500']),
@@ -81,11 +46,23 @@
 ]  # yapf: disable
 
 recent_jobs = [
-    jobConfig( name="telegraf",    targets=[f'{h}:9273' for h in all_hosts]),
-    jobConfig( name="filebeat",    targets=[f'{h}:5067' for h in all_hosts]),
-    jobConfig( name="net-routes",  targets=['pipe:9999']),
-    jobConfig( name="net-traffic", targets=['pipe:8080']),
-    jobConfig( name="dnsmasq-log", targets=['pipe:9991']),
+    jobConfig(name="dnsmasq-log", targets=['pipe:9991']),
+    jobConfig(name="filebeat",    targets=[f'{h}:5067' for h in all_hosts]),
+    jobConfig(name="net-routes",  targets=['pipe:9999']),
+    jobConfig(name="net-traffic", targets=['pipe:8080']),
+    jobConfig(name="pomerium",    targets=['pomerium-metrics.pomerium:9090']),
+    jobConfig(name="telegraf",    targets=[f'{h}:9273' for h in all_hosts]),
+    jobConfig(name="victorialogs",targets=['victorialogs'], metrics_path='/logs/metrics'),
+
+    jobConfig(name="next-victoriametrics-forever-vmagent",   metrics_path='/m/next/forever/vmagent/metrics',  targets=FromName),
+    jobConfig(name="next-victoriametrics-forever-vminsert",  metrics_path='/m/next/forever/vminsert/metrics', targets=FromName),
+    jobConfig(name="next-victoriametrics-forever-vmstorage", metrics_path='/m/next/forever/vmstorage/metrics',targets=FromName),
+    jobConfig(name="next-victoriametrics-recent-vmagent",    metrics_path='/m/next/recent/vmagent/metrics',   targets=FromName),
+    jobConfig(name="next-victoriametrics-recent-vminsert",   metrics_path='/m/next/recent/vminsert/metrics',  targets=FromName),
+    jobConfig(name="next-victoriametrics-recent-vmstorage",  metrics_path='/m/next/recent/vmstorage/metrics', targets=FromName),
+    jobConfig(name="next-victoriametrics-vmselect",          metrics_path='/m/next/vmselect/metrics',         targets=FromName),
+    jobConfig(name="next-victoriametrics-index",                                                              targets=FromName),
+
     jobConfig(
         name="racc",
         scrape_interval='30s',
@@ -97,8 +74,90 @@
         ],
     ),
 ]  # yapf: disable
-recent_jobs.extend(scrape_deployments(deploy_doesnt_serve_metrics, forever_jobs))
+
+
+deploy_doesnt_serve_metrics = [
+    'apprise',
+    'bitwarden',
+    'digi-files',
+    'digi-pose-predict',
+    'digi-tts-mimic',
+    'digi-web',
+    'dovecot',
+    'ectoscope',
+    'front-door-display',
+    'hass',
+    'homepage',
+    'itch150',
+    'jsregistry',
+    'kallithea',
+    'kube-web-view',
+    'magma',
+    'megasecond',
+    'minecraft-build-world',
+    'minecraft-lake-world',
+    'minecraft-smp-world',
+    'mongodb',
+    'mqtt1',
+    'mqtt2',
+    'nodered',
+    'photoprism',
+    'plik',
+    'projects',
+    'registry-ui',
+    'registry',
+    'speakerphone',
+    'victorialogs-ui',
+    'video-files',
+    'video',
+    'zigbee2mqtt',
+    'zwave2mqtt',
+]
+
+existing_jobs = [j['job_name'] for j in forever_jobs + recent_jobs]
+recent_jobs.extend(scrape_deployments(deploy_doesnt_serve_metrics + existing_jobs))
 
-top = Path('build/scrape_config')
-writeJobConfigs(top, forever_jobs, 'forever')
-writeJobConfigs(top, recent_jobs, 'recent')
+recent_jobs.append(jobConfig(name='kubernetes-apiservers', https=True, targets=[]) | {
+    'kubernetes_sd_configs': [{
+        'role': 'endpoints'
+    }],
+    'relabel_configs': [{
+        'source_labels': ['__meta_kubernetes_namespace', '__meta_kubernetes_service_name', '__meta_kubernetes_endpoint_port_name'],
+        'action': 'keep',
+        'regex': 'default;kubernetes;https'
+    }],
+})
+
+recent_jobs.append(
+    jobConfig(name="kubernetes-nodes", https=True, targets=[]) | {
+        "kubernetes_sd_configs": [{
+            "role": "node"
+        }],
+        "relabel_configs": [{
+            "action": "labeldrop",
+            "regex": "__meta_kubernetes_node_label_(feature_node|nvidia_com_|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*"
+        }, {
+            "action": "labelmap",
+            "regex": "__meta_kubernetes_node_label_(.+)"
+        }, {
+            "action": "labeldrop",
+            "regex": "kubernetes_io_hostname"
+        }],
+    })
+
+# see https://github.com/google/cadvisor/blob/master/docs/storage/prometheus.md
+# for metric definitions
+
+recent_jobs.append(jobConfig(name="kubernetes-cadvisor", https=True, metrics_path="/metrics/cadvisor", targets=[]) | {
+    "kubernetes_sd_configs": [{
+        "role": "node"
+    }],
+    "relabel_configs": [{
+        "action": "labeldrop",
+        "regex": "(feature_node|nvidia_com_gpu|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|beta_kubernetes_io_os|node_kubernetes_io_instance_type|kubernetes_io_os).*"
+    }],
+})
+
+outDir = Path('build/scrape_config')
+writeJobConfigs(outDir, forever_jobs, 'forever')
+writeJobConfigs(outDir, recent_jobs, 'recent')