Mercurial > code > home > repos > victoriametrics
changeset 56:a72c47973aa4
parameterize the python version
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 26 Apr 2024 16:10:47 -0700 |
| parents | 24f4ec319f98 |
| children | b3addcd4486c |
| files | next/create_all.py next/index_page.py next/output.py |
| diffstat | 3 files changed, 420 insertions(+), 252 deletions(-) [+] |
line wrap: on
line diff
--- a/next/create_all.py Fri Apr 26 15:55:25 2024 -0700 +++ b/next/create_all.py Fri Apr 26 16:10:47 2024 -0700 @@ -1,278 +1,243 @@ -import json -from pathlib import Path - - -def toJson(d): - return json.dumps(d, sort_keys=True, indent=2) +from index_page import makeIndexHtml +from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc -build = Path('build') -(build / 'vmagent_deploy.yaml').write_text( - toJson({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "next-victoriametrics-vmagent" - }, - "spec": { - "replicas": 1, - "strategy": { - "type": "Recreate" - }, - "selector": { - "matchLabels": { - "app": "next-victoriametrics-vmagent" - } - }, - "template": { - "metadata": { - "labels": { - "app": "next-victoriametrics-vmagent" +def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName): + (build / f'{agentFileName}_deploy.yaml').write_text( + toJson({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName }, + "spec": { + "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } }, + "template": { + "metadata": { + "labels": { "app": agentName }, + "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" } }, - "annotations": { - "prometheus.io/scrape": "true", - "prometheus.io/path": "/m/metrics", - "prometheus.io/port": "80" + "spec": { + "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }], + "serviceAccountName": "victoriametrics", + "containers": [{ + "name": "vmagent", + "image": f"docker.io/victoriametrics/vmagent:{vmVersion}", + "imagePullPolicy": "IfNotPresent", + "args": [ + f"-http.pathPrefix={pipelineWebRoot}/vmagent/", + tzArg, + f"-promscrape.config=/local/config/{scrapeMapKey}", + "-promscrape.configCheckInterval=5s", + "-sortLabels", + f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write", + "-remoteWrite.showURL", + ], + "ports": [{ "containerPort": agentPort }], + "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }] + }] } - }, - "spec": { - "volumes": [{ - "name": "config", - "configMap": { - "name": "victoriametrics-config" - } - }], - "serviceAccountName": - "victoriametrics", - "containers": [{ - "name": "vmagent", - "image": "docker.io/victoriametrics/vmagent:v1.100.1", - "imagePullPolicy": "IfNotPresent", - "args": ["-http.pathPrefix=/m/next/vmagent/", "-loggerTimezone=America/Los_Angeles", "-promscrape.config=/local/config/scrape_main", "-promscrape.configCheckInterval=5s", "-sortLabels", "-remoteWrite.url=http://next-victoriametrics-vminsert/m/next/vminsert/insert/0/prometheus/api/v1/write", "-remoteWrite.showURL"], - "ports": [{ - "containerPort": 8429 - }], - "volumeMounts": [{ - "name": "config", - "mountPath": "/local/config" - }] - }] } } - } - })) -(build / 'vmagent_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmagent"}, "spec": {"ports": [{"port": 80, "targetPort": 8429}], "selector": {"app": "next-victoriametrics-vmagent"}}})) + })) # yapf: disable + -(build / 'vminsert_deploy.yaml').write_text( - toJson({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "next-victoriametrics-vminsert" - }, - "spec": { - "replicas": 1, - "strategy": { - "type": "Recreate" - }, - "selector": { - "matchLabels": { - "app": "next-victoriametrics-vminsert" - } - }, - "template": { - "metadata": { - "labels": { - "app": "next-victoriametrics-vminsert" +def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort): + (build / f'{insertFileName}_deploy.yaml').write_text( + toJson({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName }, + "spec": { + "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } }, + "template": { + "metadata": { + "labels": { "app": insertName }, + "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" } }, - "annotations": { - "prometheus.io/scrape": "true", - "prometheus.io/path": "/m/metrics", - "prometheus.io/port": "80" + "spec": { + "serviceAccountName": "victoriametrics", + "containers": [{ + "name": "vminsert", + "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster", + "imagePullPolicy": "IfNotPresent", + "args": [ + f"-http.pathPrefix={pipelineWebRoot}/vminsert/", + tzArg, + f"-storageNode={storageName}", + ], + "ports": [{ "containerPort": insertPort }] + }] } - }, - "spec": { - "serviceAccountName": "victoriametrics", - "containers": [{ - "name": "vminsert", - "image": "docker.io/victoriametrics/vminsert:v1.100.1-cluster", - "imagePullPolicy": "IfNotPresent", - "args": ["-http.pathPrefix=/m/next/vminsert/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"], - "ports": [{ - "containerPort": 8480 - }] - }] } } - } - })) -(build / 'vminsert_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vminsert"}, "spec": {"ports": [{"port": 80, "targetPort": 8480}], "selector": {"app": "next-victoriametrics-vminsert"}}})) + })) # yapf: disable -(build / 'vmstorage_pv.yaml').write_text( - toJson({ - "apiVersion": "v1", - "kind": "PersistentVolume", - "metadata": { - "name": "next-victoriametrics", - "labels": { - "type": "local" - } - }, - "spec": { - "storageClassName": "manual", - "hostPath": { - "path": "/opt/next-victoriametrics" - }, - "capacity": { - "storage": "50Gi" - }, - "accessModes": ["ReadWriteMany"], - "persistentVolumeReclaimPolicy": "Retain", - "claimRef": { - "namespace": "default", - "name": "next-victoriametrics" - } - } - })) -(build / 'vmstorage_pvc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": "next-victoriametrics"}, "spec": {"storageClassName": "", "volumeName": "next-victoriametrics", "accessModes": ["ReadWriteMany"], "resources": {"requests": {"storage": "50Gi"}}}})) -(build / 'vmstorage_deploy.yaml').write_text( - toJson({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "next-victoriametrics-vmstorage" - }, - "spec": { - "replicas": 1, - "strategy": { - "type": "Recreate" - }, - "selector": { - "matchLabels": { - "app": "next-victoriametrics-vmstorage" - } - }, - "template": { - "metadata": { - "labels": { - "app": "next-victoriametrics-vmstorage" +def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort): + (build / f'{storageFileName}_2deploy.yaml').write_text( + toJson({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName }, + "spec": { + "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } }, + "template": { + "metadata": { + "labels": { "app": storageName }, + "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/vmstorage/metrics", "prometheus.io/port": "80" } }, - "annotations": { - "prometheus.io/scrape": "true", - "prometheus.io/path": "/m/vmstorage/metrics", - "prometheus.io/port": "80" - } - }, - "spec": { - "volumes": [{ - "name": "data", - "persistentVolumeClaim": { - "claimName": "next-victoriametrics" - } - }], - "serviceAccountName": "victoriametrics", - "containers": [{ - "name": "vmstorage", - "image": "docker.io/victoriametrics/vmstorage:v1.100.1-cluster", - "imagePullPolicy": "IfNotPresent", - "args": ["-http.pathPrefix=/m/next/vmstorage/", "-loggerTimezone=America/Los_Angeles", "-retentionPeriod=100y", "-storageDataPath=/data/100y"], - "ports": [{ - "containerPort": 8482, - "name": "http" - }, { - "containerPort": 8400, - "name": "vminsert" - }, { - "containerPort": 8401, - "name": "vmselect" + "spec": { + "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }], + "serviceAccountName": "victoriametrics", + "containers": [{ + "name": "vmstorage", + "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster", + "imagePullPolicy": "IfNotPresent", + "args": [ + f"-http.pathPrefix={pipelineWebRoot}/vmstorage/", + tzArg, + f"-retentionPeriod={retention}", + f"-storageDataPath=/data/{pipelineName}", + ], + "ports": [ + { "containerPort": 8482, "name": "http" }, + { "containerPort": storageInsertPort, "name": "vminsert" }, + { "containerPort": storageSelectPort, "name": "vmselect" }, + ], + "volumeMounts": [{ "name": "data", "mountPath": "/data" }] }], - "volumeMounts": [{ - "name": "data", - "mountPath": "/data" - }] - }], - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kubernetes.io/hostname", - "operator": "In", - "values": ["ditto"] - }] - }] - } - } + "affinity": affinityToNode(localPvHost) } } } - } - })) -(build / 'vmstorage_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmstorage"}, "spec": {"ports": [{"port": 80, "targetPort": "http", "name": "http"}, {"port": 8400, "targetPort": "vminsert", "name": "vminsert"}, {"port": 8401, "targetPort": "vmselect", "name": "vmselect"}], "selector": {"app": "next-victoriametrics-vmstorage"}}})) + })) # yapf: disable + -(build / 'vmselect_deploy.yaml').write_text( - toJson({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "next-victoriametrics-vmselect" - }, - "spec": { - "replicas": 1, - "strategy": { - "type": "Recreate" - }, - "selector": { - "matchLabels": { - "app": "next-victoriametrics-vmselect" - } - }, - "template": { - "metadata": { - "labels": { - "app": "next-victoriametrics-vmselect" +def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort): + name = f"{objPrefix}-vmselect" + (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text( + toJson({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name }, + "spec": { + "replicas": 1, + "strategy": { "type": "Recreate" }, + "selector": { "matchLabels": { "app": name } }, + "template": { + "metadata": { + "labels": { "app": name }, + "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" } }, - "annotations": { - "prometheus.io/scrape": "true", - "prometheus.io/path": "/m/metrics", - "prometheus.io/port": "80" + "spec": { + "serviceAccountName": "victoriametrics", + "containers": [{ + "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent", + "args": [ + f"-http.pathPrefix={webRoot}/vmselect/", + tzArg, + ] + [f"-storageNode={n}" for n in storageSvcs], + "ports": [{ "containerPort": selectPort }] + }] } - }, - "spec": { - "serviceAccountName": "victoriametrics", - "containers": [{ - "name": "vmselect", - "image": "docker.io/victoriametrics/vmselect:v1.100.1-cluster", - "imagePullPolicy": "IfNotPresent", - "args": ["-http.pathPrefix=/m/next/vmselect/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"], - "ports": [{ - "containerPort": 8481 - }] - }] } } - } - })) -(build / 'vmselect_svc.yaml').write_text(toJson({ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "next-victoriametrics-vmselect" - }, - "spec": { - "ports": [{ - "port": 80, - "targetPort": 8481 - }], - "selector": { - "app": "next-victoriametrics-vmselect" - } - } -} - # in vmui, set server url to - # https://bigasterisk.com/m/next/vmselect/select/0/prometheus - )) + })) # yapf: disable + +def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention): + agentName = f"{objPrefix}-{pipelineName}-vmagent" + insertName = f"{objPrefix}-{pipelineName}-vminsert" + storageName = f"{objPrefix}-{pipelineName}-vmstorage" + + agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent" + insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert" + storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage" + + localPvHost = "ditto" + insertPort = 8480 + agentPort = 8429 + storageInsertPort = 8400 + storageSelectPort = 8401 + volName = f"{objPrefix}-data-{pipelineName}" + request = "50Gi" + pipelineWebRoot = f'{webRoot}/{pipelineName}' + + createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName) + createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort) + createPv(storageFileName, volName, request) + createPvc(storageFileName, volName, request) + createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort) + + createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}]) + createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}]) + createSvc(storageFileName,storageName, [ + {"port": 80, "targetPort": "http", "name": "http"}, + {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"}, + {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"}, + ]) # yapf: disable + + return storageName + + +def createIndex(objPrefix, webRoot, html): + name = f'{objPrefix}-index' + httpServeRoot = '/opt/html' + + (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({ + "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name }, + "data": { + "index.html": html, +(??) } +(??) })) +(??)(build / 'vmselect_svc.yaml').write_text(toJson({ +(??) "apiVersion": "v1", +(??) "kind": "Service", +(??) "metadata": { +(??) "name": "next-victoriametrics-vmselect" +(??) }, +(??) "spec": { +(??) "ports": [{ +(??) "port": 80, +(??) "targetPort": 8481 +(??) }], +(??) "selector": { +(??) "app": "next-victoriametrics-vmselect" +(??) } +(??) } +(??)} +(??) # in vmui, set server url to +(??) # https://bigasterisk.com/m/next/vmselect/select/0/prometheus +(??) )) +(??) +(??)(build / 'vmselect_svc.yaml').write_text( +(??) toJson({ +(??) "apiVersion": "networking.k8s.io/v1", +(??) "kind": "Ingress", +(??) "metadata": { +(??) "name": "next-victoriametrics", +(??) "annotations": { +(??) "cert-manager.io/cluster-issuer": "letsencrypt-prod", +(??) "ingress.pomerium.io/allow_public_unauthenticated_access": "false", +(??) "ingress.pomerium.io/pass_identity_headers": "true", +(??) "ingress.pomerium.io/preserve_host_header": "true", + })) # yapf: disable + + (build / f'{objPrefix}-3index_deploy.yaml').write_text( + toJson({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name }, + "spec": { + "replicas": 1, + "selector": { "matchLabels": { "app": name } }, + "template": { + "metadata": { "labels": { "app": name } }, + "spec": { + "containers": [{ + "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent", + "args": [ + f'--root={httpServeRoot}', + '--directory-listing=true', + '--experimental-metrics=true', + ], + "ports": [{ "containerPort": 80 }], + "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }] + }], + "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }] + } + } + } + })) # yapf: disable + createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}]) (build / 'vmselect_svc.yaml').write_text( toJson({
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/next/index_page.py Fri Apr 26 16:10:47 2024 -0700 @@ -0,0 +1,82 @@ + + +def makeIndexHtml(objPrefix, webRoot, webHost): + unquotedJs = ''' + localStorage.setItem('SERVER_URL', JSON.stringify({value: serverUrl})); + ''' + return f"""<!DOCTYPE html> + <html> + <head> + <title>{objPrefix}</title> + <style> + section {{ margin-left: 2em; }} + h1, h2 {{ border-top: 1px solid lightgray; }} + </style> + </head> + <body> + <h1>{objPrefix}</h1> + <section> + <h2>Retentions</h2> + <section> + <h3><code>recent</code></h3> + <table> + <tr> + <td><a href="recent/vmagent/">vmagent</a></td> + <td><a href="recent/vmagent/metrics">metrics</a></td> + </tr> + <tr> + <td><a href="recent/vminsert/">vminsert</a></td> + <td><a href="recent/vminsert/metrics">metrics</a></td> + </tr> + <tr> + <td><a href="recent/vmstorage/">vmstorage</a></td> + <td><a href="recent/vmstorage/metrics">metrics</a></td> + </tr> + </table> + </section> + + <section> + <h3><code>forever</code></h3> + <table> + <tr> + <td><a href="forever/vmagent/">vmagent</a></td> + <td><a href="forever/vmagent/metrics">metrics</a></td> + </tr> + <tr> + <td><a href="forever/vminsert/">vminsert</a></td> + <td><a href="forever/vminsert/metrics">metrics</a></td> + </tr> + <tr> + <td><a href="forever/vmstorage/">vmstorage</a></td> + <td><a href="forever/vmstorage/metrics">metrics</a></td> + </tr> + </table> + </section> + </section> + + <section> + <h2>vmselect</h2> + <table> + <tr> + <td><a href="vmselect/">vmselect</a></td> + <td><a href="vmselect/metrics">metrics</a></td> + </tr> + </table> + </section> + + <section> + <h2>vmui</h2> + <table> + <tr> + <td><a href="vmselect/0/vmui/vmui">vmui</a></td> + </tr> + </table> + </section> + + <script> + // this defaults to something incorrect, so we fix it hopefully before you go to vmui + let serverUrl = "https://{webHost}{webRoot}/vmselect/select/0/prometheus"; + {unquotedJs} + </script> + </body> + </html>""" \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/next/output.py Fri Apr 26 16:10:47 2024 -0700 @@ -0,0 +1,121 @@ +import json +from pathlib import Path + +build = Path('build') + + +def toJson(d): + return json.dumps(d, sort_keys=True, indent=2) + + +def createSvc(filename, name, ports): + (build / f'{filename}_svc.yaml').write_text(toJson({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": name + }, + "spec": { + "ports": ports, + "selector": { + "app": name + } + }, + })) + + +def createIngress(filename, objName, policy, ingressPaths, host): + + (build / filename).write_text( + toJson({ + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "name": objName, + "annotations": { + "cert-manager.io/cluster-issuer": "letsencrypt-prod", + "ingress.pomerium.io/allow_public_unauthenticated_access": "false", + "ingress.pomerium.io/pass_identity_headers": "true", + "ingress.pomerium.io/preserve_host_header": "true", + "ingress.pomerium.io/policy": policy, + } + }, + "spec": { + "ingressClassName": "pomerium", + "rules": [{ + "host": host, + "http": { + "paths": ingressPaths + } + },], + "tls": [{ + "hosts": [host], + "secretName": f"{host}-tls" + }] + } + })) + + +def createPv(storageFileName, volName, request): + (build / f'{storageFileName}_0pv.yaml').write_text( + toJson({ + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": { + "name": volName, + "labels": { + "type": "local" + } + }, + "spec": { + "storageClassName": "manual", + "hostPath": { + "path": f"/opt/{volName}" + }, + "capacity": { + "storage": request + }, + "accessModes": ["ReadWriteMany"], + "persistentVolumeReclaimPolicy": "Retain", + "claimRef": { + "namespace": "default", + "name": volName + } + } + })) + + +def createPvc(storageFileName, volName, request): + (build / f'{storageFileName}_1pvc.yaml').write_text(toJson({ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": volName, + }, + "spec": { + "storageClassName": "", + "volumeName": volName, + "accessModes": ["ReadWriteMany"], + "resources": { + "requests": { + "storage": request + } + } + }, + })) + + +def affinityToNode(node): + return { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{ + "matchExpressions": [{ + "key": "kubernetes.io/hostname", + "operator": "In", + "values": [node], + }], + }], + }, + } + }