--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/create_k8s.py	Fri May 03 11:21:08 2024 -0700
@@ -0,0 +1,254 @@
+from pathlib import Path
+from index_page import makeIndexHtml
+from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc
+
+
+def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix):
+    (build / f'{agentFileName}_deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName },
+            "spec": {
+                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } },
+                "template": {
+                    "metadata": {
+                        "labels": { "app": agentName },
+                        "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" }
+                    },
+                    "spec": {
+                        "volumes": [{ "name": "config", "configMap": { "name": f"{objPrefix}-config" } }],
+                        "serviceAccountName": "victoriametrics",
+                        "containers": [{
+                            "name": "vmagent",
+                            "image": f"docker.io/victoriametrics/vmagent:{vmVersion}",
+                            "imagePullPolicy": "IfNotPresent",
+                            "args": [
+                                f"-http.pathPrefix={pipelineWebRoot}/vmagent/",
+                                tzArg,
+                                f"-promscrape.config=/local/config/{scrapeMapKey}",
+                                "-promscrape.configCheckInterval=5s",
+                                "-sortLabels",
+                                f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write",
+                                "-remoteWrite.showURL",
+                            ],
+                            "ports": [{ "containerPort": agentPort }],
+                            "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }]
+                        }]
+                    }
+                }
+            }
+        })) # yapf: disable
+
+
+def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort):
+    (build / f'{insertFileName}_deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName },
+            "spec": {
+                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } },
+                "template": {
+                    "metadata": {
+                        "labels": { "app": insertName },
+                        "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
+                    },
+                    "spec": {
+                        "serviceAccountName": "victoriametrics",
+                        "containers": [{
+                            "name": "vminsert",
+                            "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster",
+                            "imagePullPolicy": "IfNotPresent",
+                            "args": [
+                                f"-http.pathPrefix={pipelineWebRoot}/vminsert/",
+                                tzArg,
+                                f"-storageNode={storageName}",
+                            ],
+                            "ports": [{ "containerPort": insertPort }]
+                        }]
+                    }
+                }
+            }
+        })) # yapf: disable
+
+
+def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort):
+    (build / f'{storageFileName}_2deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName },
+            "spec": {
+                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } },
+                "template": {
+                    "metadata": {
+                        "labels": { "app": storageName },
+                        "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/vmstorage/metrics", "prometheus.io/port": "80" }
+                    },
+                    "spec": {
+                        "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }],
+                        "serviceAccountName": "victoriametrics",
+                        "containers": [{
+                            "name": "vmstorage",
+                            "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster",
+                            "imagePullPolicy": "IfNotPresent",
+                            "args": [
+                                f"-http.pathPrefix={pipelineWebRoot}/vmstorage/",
+                                tzArg,
+                                f"-retentionPeriod={retention}",
+                                f"-storageDataPath=/data/{pipelineName}",
+                            ],
+                            "ports": [
+                                { "containerPort": 8482, "name": "http" },
+                                { "containerPort": storageInsertPort, "name": "vminsert" },
+                                { "containerPort": storageSelectPort, "name": "vmselect" },
+                            ],
+                            "volumeMounts": [{ "name": "data", "mountPath": "/data" }]
+                        }],
+                        "affinity": affinityToNode(localPvHost)
+                    }
+                }
+            }
+        })) # yapf: disable
+
+
+def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort):
+    name = f"{objPrefix}-vmselect"
+    (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
+            "spec": {
+                "replicas": 1,
+                "strategy": { "type": "Recreate" },
+                "selector": { "matchLabels": { "app": name } },
+                "template": {
+                    "metadata": {
+                        "labels": { "app": name },
+                        "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
+                    },
+                    "spec": {
+                        "serviceAccountName": "victoriametrics",
+                        "containers": [{
+                            "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent",
+                            "args": [
+                                f"-http.pathPrefix={webRoot}/vmselect/",
+                                tzArg,
+                            ] + [f"-storageNode={n}" for n in storageSvcs],
+                            "ports": [{ "containerPort": selectPort }]
+                        }]
+                    }
+                }
+            }
+        })) # yapf: disable
+
+def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention):
+    agentName = f"{objPrefix}-{pipelineName}-vmagent"
+    insertName = f"{objPrefix}-{pipelineName}-vminsert"
+    storageName = f"{objPrefix}-{pipelineName}-vmstorage"
+
+    agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent"
+    insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert"
+    storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage"
+
+    localPvHost = "ditto"
+    insertPort = 8480
+    agentPort = 8429
+    storageInsertPort = 8400
+    storageSelectPort = 8401
+    volName = f"{objPrefix}-data-{pipelineName}"
+    request = "50Gi"
+    pipelineWebRoot = f'{webRoot}/{pipelineName}'
+
+    createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix)
+    createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort)
+    createPv(storageFileName, volName, request)
+    createPvc(storageFileName, volName, request)
+    createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort)
+
+    createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}])
+    createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}])
+    createSvc(storageFileName,storageName, [
+        {"port": 80, "targetPort": "http", "name": "http"},
+        {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"},
+        {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"},
+        ]) # yapf: disable
+
+    return storageName
+
+
+def createIndex(objPrefix, webRoot, html):
+    name = f'{objPrefix}-index'
+    httpServeRoot = '/opt/html'
+
+    (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({
+        "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name },
+        "data": {
+            "index.html": html,
+            "index.js": Path("index.js").read_text(),
+            "index.css": Path("index.css").read_text(),
+        }
+    })) # yapf: disable
+
+    (build / f'{objPrefix}-3index_deploy.yaml').write_text(
+        toJson({
+            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
+            "spec": {
+                "replicas": 1,
+                "selector": { "matchLabels": { "app": name } },
+                "template": {
+                    "metadata": { "labels": { "app": name } },
+                    "spec": {
+                        "containers": [{
+                            "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent",
+                            "args": [
+                                f'--root={httpServeRoot}',
+                                '--directory-listing=true',
+                                '--experimental-metrics=true',
+                            ],
+                            "ports": [{ "containerPort": 80 }],
+                            "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }]
+                        }],
+                        "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }]
+                    }
+                }
+            }
+        })) # yapf: disable
+    createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}])
+
+
+def main():
+    tzArg = "-loggerTimezone=America/Los_Angeles"
+    objPrefix = "next-victoriametrics"  # prefix on all k8s object names
+    webRoot = "/m/next"
+    vmVersion = "v1.100.1"
+    webHost = 'bigasterisk.com'
+    pipelines = [
+        ('forever', '100y'),
+        ('recent', '90y'),
+    ]
+    storageSvcs = [createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, p, f'scrape_{p}.yaml', ret) for p, ret in pipelines]
+
+    selectPort = 8481
+    createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort)
+    createSvc(f'{objPrefix}-1vmselect', f"{objPrefix}-vmselect", [{"port": 80, "targetPort": selectPort}])
+
+    ingressPaths = [
+        { "pathType": "Prefix", "path": f"{webRoot}/",          "backend": { "service": { "name": f"{objPrefix}-index", "port": { "number": 80 } } } },
+        { "pathType": "Prefix", "path": f"{webRoot}/vmselect/", "backend": { "service": { "name": f"{objPrefix}-vmselect", "port": { "number": 80 } } } },
+    ]  # yapf: disable
+    for p, _ in pipelines:
+        ingressPaths.extend([
+            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vmagent/",   "backend": { "service": { "name": f"{objPrefix}-{p}-vmagent",   "port": { "number": 80 } } } },
+            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vminsert/",  "backend": { "service": { "name": f"{objPrefix}-{p}-vminsert",  "port": { "number": 80 } } } },
+            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vmstorage/", "backend": { "service": { "name": f"{objPrefix}-{p}-vmstorage", "port": { "number": 80 } } } },
+        ]) # yapf: disable
+
+    policy = """\
+allow:
+    or: 
+        - { email: { is: "drewpca@gmail.com" }}
+        - { email: { is: "kelsimp@gmail.com" }}
+    """
+    createIngress(f'{objPrefix}-2ingress.yaml', objPrefix, policy, ingressPaths, webHost)
+    createIndex(objPrefix, webRoot, makeIndexHtml(objPrefix, webRoot, webHost))
+
+
+main()
+
+# in vmui, set server url to
+# https://bigasterisk.com{webRoot}/vmselect/select/0/prometheus