# HG changeset patch
# User drewp@bigasterisk.com
# Date 1714173047 25200
# Node ID a72c47973aa4b0a1dfa0aaddd96754c85b15e5b8
# Parent 24f4ec319f98e455b6c4710611d6d7e3f4199a6a
parameterize the python version
diff -r 24f4ec319f98 -r a72c47973aa4 next/create_all.py
--- a/next/create_all.py Fri Apr 26 15:55:25 2024 -0700
+++ b/next/create_all.py Fri Apr 26 16:10:47 2024 -0700
@@ -1,278 +1,243 @@
-import json
-from pathlib import Path
-
-
-def toJson(d):
- return json.dumps(d, sort_keys=True, indent=2)
+from index_page import makeIndexHtml
+from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc
-build = Path('build')
-(build / 'vmagent_deploy.yaml').write_text(
- toJson({
- "apiVersion": "apps/v1",
- "kind": "Deployment",
- "metadata": {
- "name": "next-victoriametrics-vmagent"
- },
- "spec": {
- "replicas": 1,
- "strategy": {
- "type": "Recreate"
- },
- "selector": {
- "matchLabels": {
- "app": "next-victoriametrics-vmagent"
- }
- },
- "template": {
- "metadata": {
- "labels": {
- "app": "next-victoriametrics-vmagent"
+def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName):
+ (build / f'{agentFileName}_deploy.yaml').write_text(
+ toJson({
+ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName },
+ "spec": {
+ "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } },
+ "template": {
+ "metadata": {
+ "labels": { "app": agentName },
+ "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" }
},
- "annotations": {
- "prometheus.io/scrape": "true",
- "prometheus.io/path": "/m/metrics",
- "prometheus.io/port": "80"
+ "spec": {
+ "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }],
+ "serviceAccountName": "victoriametrics",
+ "containers": [{
+ "name": "vmagent",
+ "image": f"docker.io/victoriametrics/vmagent:{vmVersion}",
+ "imagePullPolicy": "IfNotPresent",
+ "args": [
+ f"-http.pathPrefix={pipelineWebRoot}/vmagent/",
+ tzArg,
+ f"-promscrape.config=/local/config/{scrapeMapKey}",
+ "-promscrape.configCheckInterval=5s",
+ "-sortLabels",
+ f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write",
+ "-remoteWrite.showURL",
+ ],
+ "ports": [{ "containerPort": agentPort }],
+ "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }]
+ }]
}
- },
- "spec": {
- "volumes": [{
- "name": "config",
- "configMap": {
- "name": "victoriametrics-config"
- }
- }],
- "serviceAccountName":
- "victoriametrics",
- "containers": [{
- "name": "vmagent",
- "image": "docker.io/victoriametrics/vmagent:v1.100.1",
- "imagePullPolicy": "IfNotPresent",
- "args": ["-http.pathPrefix=/m/next/vmagent/", "-loggerTimezone=America/Los_Angeles", "-promscrape.config=/local/config/scrape_main", "-promscrape.configCheckInterval=5s", "-sortLabels", "-remoteWrite.url=http://next-victoriametrics-vminsert/m/next/vminsert/insert/0/prometheus/api/v1/write", "-remoteWrite.showURL"],
- "ports": [{
- "containerPort": 8429
- }],
- "volumeMounts": [{
- "name": "config",
- "mountPath": "/local/config"
- }]
- }]
}
}
- }
- }))
-(build / 'vmagent_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmagent"}, "spec": {"ports": [{"port": 80, "targetPort": 8429}], "selector": {"app": "next-victoriametrics-vmagent"}}}))
+ })) # yapf: disable
+
-(build / 'vminsert_deploy.yaml').write_text(
- toJson({
- "apiVersion": "apps/v1",
- "kind": "Deployment",
- "metadata": {
- "name": "next-victoriametrics-vminsert"
- },
- "spec": {
- "replicas": 1,
- "strategy": {
- "type": "Recreate"
- },
- "selector": {
- "matchLabels": {
- "app": "next-victoriametrics-vminsert"
- }
- },
- "template": {
- "metadata": {
- "labels": {
- "app": "next-victoriametrics-vminsert"
+def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort):
+ (build / f'{insertFileName}_deploy.yaml').write_text(
+ toJson({
+ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName },
+ "spec": {
+ "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } },
+ "template": {
+ "metadata": {
+ "labels": { "app": insertName },
+ "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
},
- "annotations": {
- "prometheus.io/scrape": "true",
- "prometheus.io/path": "/m/metrics",
- "prometheus.io/port": "80"
+ "spec": {
+ "serviceAccountName": "victoriametrics",
+ "containers": [{
+ "name": "vminsert",
+ "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster",
+ "imagePullPolicy": "IfNotPresent",
+ "args": [
+ f"-http.pathPrefix={pipelineWebRoot}/vminsert/",
+ tzArg,
+ f"-storageNode={storageName}",
+ ],
+ "ports": [{ "containerPort": insertPort }]
+ }]
}
- },
- "spec": {
- "serviceAccountName": "victoriametrics",
- "containers": [{
- "name": "vminsert",
- "image": "docker.io/victoriametrics/vminsert:v1.100.1-cluster",
- "imagePullPolicy": "IfNotPresent",
- "args": ["-http.pathPrefix=/m/next/vminsert/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"],
- "ports": [{
- "containerPort": 8480
- }]
- }]
}
}
- }
- }))
-(build / 'vminsert_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vminsert"}, "spec": {"ports": [{"port": 80, "targetPort": 8480}], "selector": {"app": "next-victoriametrics-vminsert"}}}))
+ })) # yapf: disable
-(build / 'vmstorage_pv.yaml').write_text(
- toJson({
- "apiVersion": "v1",
- "kind": "PersistentVolume",
- "metadata": {
- "name": "next-victoriametrics",
- "labels": {
- "type": "local"
- }
- },
- "spec": {
- "storageClassName": "manual",
- "hostPath": {
- "path": "/opt/next-victoriametrics"
- },
- "capacity": {
- "storage": "50Gi"
- },
- "accessModes": ["ReadWriteMany"],
- "persistentVolumeReclaimPolicy": "Retain",
- "claimRef": {
- "namespace": "default",
- "name": "next-victoriametrics"
- }
- }
- }))
-(build / 'vmstorage_pvc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": "next-victoriametrics"}, "spec": {"storageClassName": "", "volumeName": "next-victoriametrics", "accessModes": ["ReadWriteMany"], "resources": {"requests": {"storage": "50Gi"}}}}))
-(build / 'vmstorage_deploy.yaml').write_text(
- toJson({
- "apiVersion": "apps/v1",
- "kind": "Deployment",
- "metadata": {
- "name": "next-victoriametrics-vmstorage"
- },
- "spec": {
- "replicas": 1,
- "strategy": {
- "type": "Recreate"
- },
- "selector": {
- "matchLabels": {
- "app": "next-victoriametrics-vmstorage"
- }
- },
- "template": {
- "metadata": {
- "labels": {
- "app": "next-victoriametrics-vmstorage"
+def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort):
+ (build / f'{storageFileName}_2deploy.yaml').write_text(
+ toJson({
+ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName },
+ "spec": {
+ "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } },
+ "template": {
+ "metadata": {
+ "labels": { "app": storageName },
+ "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/vmstorage/metrics", "prometheus.io/port": "80" }
},
- "annotations": {
- "prometheus.io/scrape": "true",
- "prometheus.io/path": "/m/vmstorage/metrics",
- "prometheus.io/port": "80"
- }
- },
- "spec": {
- "volumes": [{
- "name": "data",
- "persistentVolumeClaim": {
- "claimName": "next-victoriametrics"
- }
- }],
- "serviceAccountName": "victoriametrics",
- "containers": [{
- "name": "vmstorage",
- "image": "docker.io/victoriametrics/vmstorage:v1.100.1-cluster",
- "imagePullPolicy": "IfNotPresent",
- "args": ["-http.pathPrefix=/m/next/vmstorage/", "-loggerTimezone=America/Los_Angeles", "-retentionPeriod=100y", "-storageDataPath=/data/100y"],
- "ports": [{
- "containerPort": 8482,
- "name": "http"
- }, {
- "containerPort": 8400,
- "name": "vminsert"
- }, {
- "containerPort": 8401,
- "name": "vmselect"
+ "spec": {
+ "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }],
+ "serviceAccountName": "victoriametrics",
+ "containers": [{
+ "name": "vmstorage",
+ "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster",
+ "imagePullPolicy": "IfNotPresent",
+ "args": [
+ f"-http.pathPrefix={pipelineWebRoot}/vmstorage/",
+ tzArg,
+ f"-retentionPeriod={retention}",
+ f"-storageDataPath=/data/{pipelineName}",
+ ],
+ "ports": [
+ { "containerPort": 8482, "name": "http" },
+ { "containerPort": storageInsertPort, "name": "vminsert" },
+ { "containerPort": storageSelectPort, "name": "vmselect" },
+ ],
+ "volumeMounts": [{ "name": "data", "mountPath": "/data" }]
}],
- "volumeMounts": [{
- "name": "data",
- "mountPath": "/data"
- }]
- }],
- "affinity": {
- "nodeAffinity": {
- "requiredDuringSchedulingIgnoredDuringExecution": {
- "nodeSelectorTerms": [{
- "matchExpressions": [{
- "key": "kubernetes.io/hostname",
- "operator": "In",
- "values": ["ditto"]
- }]
- }]
- }
- }
+ "affinity": affinityToNode(localPvHost)
}
}
}
- }
- }))
-(build / 'vmstorage_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmstorage"}, "spec": {"ports": [{"port": 80, "targetPort": "http", "name": "http"}, {"port": 8400, "targetPort": "vminsert", "name": "vminsert"}, {"port": 8401, "targetPort": "vmselect", "name": "vmselect"}], "selector": {"app": "next-victoriametrics-vmstorage"}}}))
+ })) # yapf: disable
+
-(build / 'vmselect_deploy.yaml').write_text(
- toJson({
- "apiVersion": "apps/v1",
- "kind": "Deployment",
- "metadata": {
- "name": "next-victoriametrics-vmselect"
- },
- "spec": {
- "replicas": 1,
- "strategy": {
- "type": "Recreate"
- },
- "selector": {
- "matchLabels": {
- "app": "next-victoriametrics-vmselect"
- }
- },
- "template": {
- "metadata": {
- "labels": {
- "app": "next-victoriametrics-vmselect"
+def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort):
+ name = f"{objPrefix}-vmselect"
+ (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text(
+ toJson({
+ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
+ "spec": {
+ "replicas": 1,
+ "strategy": { "type": "Recreate" },
+ "selector": { "matchLabels": { "app": name } },
+ "template": {
+ "metadata": {
+ "labels": { "app": name },
+ "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
},
- "annotations": {
- "prometheus.io/scrape": "true",
- "prometheus.io/path": "/m/metrics",
- "prometheus.io/port": "80"
+ "spec": {
+ "serviceAccountName": "victoriametrics",
+ "containers": [{
+ "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent",
+ "args": [
+ f"-http.pathPrefix={webRoot}/vmselect/",
+ tzArg,
+ ] + [f"-storageNode={n}" for n in storageSvcs],
+ "ports": [{ "containerPort": selectPort }]
+ }]
}
- },
- "spec": {
- "serviceAccountName": "victoriametrics",
- "containers": [{
- "name": "vmselect",
- "image": "docker.io/victoriametrics/vmselect:v1.100.1-cluster",
- "imagePullPolicy": "IfNotPresent",
- "args": ["-http.pathPrefix=/m/next/vmselect/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"],
- "ports": [{
- "containerPort": 8481
- }]
- }]
}
}
- }
- }))
-(build / 'vmselect_svc.yaml').write_text(toJson({
- "apiVersion": "v1",
- "kind": "Service",
- "metadata": {
- "name": "next-victoriametrics-vmselect"
- },
- "spec": {
- "ports": [{
- "port": 80,
- "targetPort": 8481
- }],
- "selector": {
- "app": "next-victoriametrics-vmselect"
- }
- }
-}
- # in vmui, set server url to
- # https://bigasterisk.com/m/next/vmselect/select/0/prometheus
- ))
+ })) # yapf: disable
+
+def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention):
+ agentName = f"{objPrefix}-{pipelineName}-vmagent"
+ insertName = f"{objPrefix}-{pipelineName}-vminsert"
+ storageName = f"{objPrefix}-{pipelineName}-vmstorage"
+
+ agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent"
+ insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert"
+ storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage"
+
+ localPvHost = "ditto"
+ insertPort = 8480
+ agentPort = 8429
+ storageInsertPort = 8400
+ storageSelectPort = 8401
+ volName = f"{objPrefix}-data-{pipelineName}"
+ request = "50Gi"
+ pipelineWebRoot = f'{webRoot}/{pipelineName}'
+
+ createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName)
+ createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort)
+ createPv(storageFileName, volName, request)
+ createPvc(storageFileName, volName, request)
+ createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort)
+
+ createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}])
+ createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}])
+ createSvc(storageFileName,storageName, [
+ {"port": 80, "targetPort": "http", "name": "http"},
+ {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"},
+ {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"},
+ ]) # yapf: disable
+
+ return storageName
+
+
+def createIndex(objPrefix, webRoot, html):
+ name = f'{objPrefix}-index'
+ httpServeRoot = '/opt/html'
+
+ (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({
+ "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name },
+ "data": {
+ "index.html": html,
+(??) }
+(??) }))
+(??)(build / 'vmselect_svc.yaml').write_text(toJson({
+(??) "apiVersion": "v1",
+(??) "kind": "Service",
+(??) "metadata": {
+(??) "name": "next-victoriametrics-vmselect"
+(??) },
+(??) "spec": {
+(??) "ports": [{
+(??) "port": 80,
+(??) "targetPort": 8481
+(??) }],
+(??) "selector": {
+(??) "app": "next-victoriametrics-vmselect"
+(??) }
+(??) }
+(??)}
+(??) # in vmui, set server url to
+(??) # https://bigasterisk.com/m/next/vmselect/select/0/prometheus
+(??) ))
+(??)
+(??)(build / 'vmselect_svc.yaml').write_text(
+(??) toJson({
+(??) "apiVersion": "networking.k8s.io/v1",
+(??) "kind": "Ingress",
+(??) "metadata": {
+(??) "name": "next-victoriametrics",
+(??) "annotations": {
+(??) "cert-manager.io/cluster-issuer": "letsencrypt-prod",
+(??) "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
+(??) "ingress.pomerium.io/pass_identity_headers": "true",
+(??) "ingress.pomerium.io/preserve_host_header": "true",
+ })) # yapf: disable
+
+ (build / f'{objPrefix}-3index_deploy.yaml').write_text(
+ toJson({
+ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
+ "spec": {
+ "replicas": 1,
+ "selector": { "matchLabels": { "app": name } },
+ "template": {
+ "metadata": { "labels": { "app": name } },
+ "spec": {
+ "containers": [{
+ "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent",
+ "args": [
+ f'--root={httpServeRoot}',
+ '--directory-listing=true',
+ '--experimental-metrics=true',
+ ],
+ "ports": [{ "containerPort": 80 }],
+ "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }]
+ }],
+ "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }]
+ }
+ }
+ }
+ })) # yapf: disable
+ createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}])
(build / 'vmselect_svc.yaml').write_text(
toJson({
diff -r 24f4ec319f98 -r a72c47973aa4 next/index_page.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/next/index_page.py Fri Apr 26 16:10:47 2024 -0700
@@ -0,0 +1,82 @@
+
+
+def makeIndexHtml(objPrefix, webRoot, webHost):
+ unquotedJs = '''
+ localStorage.setItem('SERVER_URL', JSON.stringify({value: serverUrl}));
+ '''
+ return f"""
+
+
+ {objPrefix}
+
+
+
+ {objPrefix}
+
+
+
+
+
+
+
+
+ """
\ No newline at end of file
diff -r 24f4ec319f98 -r a72c47973aa4 next/output.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/next/output.py Fri Apr 26 16:10:47 2024 -0700
@@ -0,0 +1,121 @@
+import json
+from pathlib import Path
+
+build = Path('build')
+
+
+def toJson(d):
+ return json.dumps(d, sort_keys=True, indent=2)
+
+
+def createSvc(filename, name, ports):
+ (build / f'{filename}_svc.yaml').write_text(toJson({
+ "apiVersion": "v1",
+ "kind": "Service",
+ "metadata": {
+ "name": name
+ },
+ "spec": {
+ "ports": ports,
+ "selector": {
+ "app": name
+ }
+ },
+ }))
+
+
+def createIngress(filename, objName, policy, ingressPaths, host):
+
+ (build / filename).write_text(
+ toJson({
+ "apiVersion": "networking.k8s.io/v1",
+ "kind": "Ingress",
+ "metadata": {
+ "name": objName,
+ "annotations": {
+ "cert-manager.io/cluster-issuer": "letsencrypt-prod",
+ "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
+ "ingress.pomerium.io/pass_identity_headers": "true",
+ "ingress.pomerium.io/preserve_host_header": "true",
+ "ingress.pomerium.io/policy": policy,
+ }
+ },
+ "spec": {
+ "ingressClassName": "pomerium",
+ "rules": [{
+ "host": host,
+ "http": {
+ "paths": ingressPaths
+ }
+ },],
+ "tls": [{
+ "hosts": [host],
+ "secretName": f"{host}-tls"
+ }]
+ }
+ }))
+
+
+def createPv(storageFileName, volName, request):
+ (build / f'{storageFileName}_0pv.yaml').write_text(
+ toJson({
+ "apiVersion": "v1",
+ "kind": "PersistentVolume",
+ "metadata": {
+ "name": volName,
+ "labels": {
+ "type": "local"
+ }
+ },
+ "spec": {
+ "storageClassName": "manual",
+ "hostPath": {
+ "path": f"/opt/{volName}"
+ },
+ "capacity": {
+ "storage": request
+ },
+ "accessModes": ["ReadWriteMany"],
+ "persistentVolumeReclaimPolicy": "Retain",
+ "claimRef": {
+ "namespace": "default",
+ "name": volName
+ }
+ }
+ }))
+
+
+def createPvc(storageFileName, volName, request):
+ (build / f'{storageFileName}_1pvc.yaml').write_text(toJson({
+ "apiVersion": "v1",
+ "kind": "PersistentVolumeClaim",
+ "metadata": {
+ "name": volName,
+ },
+ "spec": {
+ "storageClassName": "",
+ "volumeName": volName,
+ "accessModes": ["ReadWriteMany"],
+ "resources": {
+ "requests": {
+ "storage": request
+ }
+ }
+ },
+ }))
+
+
+def affinityToNode(node):
+ return {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [{
+ "matchExpressions": [{
+ "key": "kubernetes.io/hostname",
+ "operator": "In",
+ "values": [node],
+ }],
+ }],
+ },
+ }
+ }