--- a/deploy_vmetrics.yaml	Sat Jun 24 01:43:55 2023 -0700
+++ b/deploy_vmetrics.yaml	Sat Jun 24 01:44:57 2023 -0700
@@ -27,12 +27,12 @@
       containers:
         - name: victoriametrics
           # https://hub.docker.com/r/victoriametrics/victoria-metrics/tags also check vmalert.yaml
-          image: bang5:5000/victoriametrics
+          image: docker.io/victoriametrics/victoria-metrics:v1.91.2
           args:
             - -http.pathPrefix=/m/
             - -loggerTimezone=America/Los_Angeles
             - -memory.allowedBytes=512MB
-            - -promscrape.config=/local/config/scrape_main.yaml
+            - -promscrape.config=/local/config/scrape_main
             - -promscrape.configCheckInterval=5s
             - -retentionPeriod=10y
             - -sortLabels
@@ -41,6 +41,7 @@
             - containerPort: 8428
           volumeMounts:
             - { name: data, mountPath: /data }
+            - { name: config, mountPath: "/local/config" }
           # resources:
           #   limits:
           #     memory: 0.5Gi

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/k8s_ops.py	Sat Jun 24 01:44:57 2023 -0700
@@ -0,0 +1,46 @@
+import json
+import time
+
+from kubernetes import client
+
+
+def refreshPodCmaps(pod_name, namespace='default'):
+    """
+    Per https://ahmet.im/blog/kubernetes-secret-volumes-delay/ there could be a while
+    until k8s updates the CM volume that a pod sees. Workaround is to edit the pod annotations.
+    """
+    api_instance = client.CoreV1Api()
+
+    pod = api_instance.read_namespaced_pod(name=pod_name, namespace=namespace)
+    if pod.metadata.annotations is None:
+        pod.metadata.annotations = {}
+    pod.metadata.annotations['force-configmap-update'] = str(time.time())
+    api_instance.replace_namespaced_pod(name=pod_name, namespace=namespace, body=pod)
+
+
+def firstPodName(selector):
+    api_instance = client.CoreV1Api()
+    pod_list = api_instance.list_namespaced_pod(namespace='default', label_selector=selector)
+    return pod_list.items[0].metadata.name
+
+
+def hup(ctx, deployment, process_name):
+    ctx.run(f'kubectl exec {deployment} -- pkill -HUP {process_name}')
+
+
+def replaceCmap(name, configObj):
+    api_instance = client.CoreV1Api()
+    api_response = api_instance.replace_namespaced_config_map(  #
+        name=name,
+        namespace='default',
+        body={
+            'apiVersion': 'v1',
+            'kind': 'ConfigMap',
+            'metadata': {
+                'name': name
+            },
+            'data': {
+                'scrape_main': json.dumps(configObj)
+            },
+        })
+    print(f"{name} resource_version is now {api_response.metadata.resource_version}")

--- a/tasks.py	Sat Jun 24 01:43:55 2023 -0700
+++ b/tasks.py	Sat Jun 24 01:44:57 2023 -0700
@@ -6,12 +6,14 @@
 from pathlib import Path
 from typing import Dict
 
+import yaml
 from invoke import task
+from kubernetes import config
 
-sys.path.append('/usr/lib/python3/dist-packages/')
-import yaml
+from k8s_ops import firstPodName, replaceCmap, refreshPodCmaps
 
 _tfs = []
+config.load_kube_config()
 
 
 def saveTmp(text):
@@ -40,7 +42,6 @@
         time.sleep(0)
 
 
-
 def hostsExpectedOnline(ctx):
     return ctx.run(
         'cd /my/serv/lanscape; pdm run python hosts_expected_online.py').stdout
@@ -85,7 +86,27 @@
         # reload(ctx, 'vmalert')
         ctx.run('kubectl rollout restart deploy/vmalert')
 
+
 @task
 def build_config(ctx):
     with open('rules/build/expected_hosts.yaml', 'w') as out:
         out.write(hostsExpectedOnline(ctx))
+
+
+# --------------------------
+
+
+def scrapeConfig(ctx):
+    return yaml.load(open('config/scrape_main.yaml'), yaml.FullLoader)
+
+
+@task
+def updateScrapes(ctx):
+    configObj = scrapeConfig(ctx)
+
+    replaceCmap('victoriametrics-config', configObj)
+
+    refreshPodCmaps(firstPodName('app=victoriametrics'))
+
+    # If the VM reloader isn't fast enough, we could do this too:
+    # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod')