Mercurial > code > home > repos > victoriametrics

comparison next/create_all.py @ 56:a72c47973aa4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
parameterize the python version
author drewp@bigasterisk.com
date Fri, 26 Apr 2024 16:10:47 -0700
parents 24f4ec319f98
children b3addcd4486c
comparison
equal deleted inserted replaced
55:24f4ec319f98 56:a72c47973aa4
1 import json 1 from index_page import makeIndexHtml
2 from pathlib import Path 2 from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc
3 3
4 4
5 def toJson(d): 5 def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName):
6 return json.dumps(d, sort_keys=True, indent=2) 6 (build / f'{agentFileName}_deploy.yaml').write_text(
7 7 toJson({
8 8 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName },
9 build = Path('build') 9 "spec": {
10 (build / 'vmagent_deploy.yaml').write_text( 10 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } },
11 toJson({ 11 "template": {
12 "apiVersion": "apps/v1", 12 "metadata": {
13 "kind": "Deployment", 13 "labels": { "app": agentName },
14 "metadata": { 14 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" }
15 "name": "next-victoriametrics-vmagent"
16 },
17 "spec": {
18 "replicas": 1,
19 "strategy": {
20 "type": "Recreate"
21 },
22 "selector": {
23 "matchLabels": {
24 "app": "next-victoriametrics-vmagent"
25 }
26 },
27 "template": {
28 "metadata": {
29 "labels": {
30 "app": "next-victoriametrics-vmagent"
31 }, 15 },
32 "annotations": { 16 "spec": {
33 "prometheus.io/scrape": "true", 17 "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }],
34 "prometheus.io/path": "/m/metrics", 18 "serviceAccountName": "victoriametrics",
35 "prometheus.io/port": "80" 19 "containers": [{
36 } 20 "name": "vmagent",
37 }, 21 "image": f"docker.io/victoriametrics/vmagent:{vmVersion}",
38 "spec": { 22 "imagePullPolicy": "IfNotPresent",
39 "volumes": [{ 23 "args": [
40 "name": "config", 24 f"-http.pathPrefix={pipelineWebRoot}/vmagent/",
41 "configMap": { 25 tzArg,
42 "name": "victoriametrics-config" 26 f"-promscrape.config=/local/config/{scrapeMapKey}",
43 } 27 "-promscrape.configCheckInterval=5s",
44 }], 28 "-sortLabels",
45 "serviceAccountName": 29 f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write",
46 "victoriametrics", 30 "-remoteWrite.showURL",
47 "containers": [{ 31 ],
48 "name": "vmagent", 32 "ports": [{ "containerPort": agentPort }],
49 "image": "docker.io/victoriametrics/vmagent:v1.100.1", 33 "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }]
50 "imagePullPolicy": "IfNotPresent", 34 }]
51 "args": ["-http.pathPrefix=/m/next/vmagent/", "-loggerTimezone=America/Los_Angeles", "-promscrape.config=/local/config/scrape_main", "-promscrape.configCheckInterval=5s", "-sortLabels", "-remoteWrite.url=http://next-victoriametrics-vminsert/m/next/vminsert/insert/0/prometheus/api/v1/write", "-remoteWrite.showURL"], 35 }
52 "ports": [{ 36 }
53 "containerPort": 8429 37 }
38 })) # yapf: disable
39
40
41 def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort):
42 (build / f'{insertFileName}_deploy.yaml').write_text(
43 toJson({
44 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName },
45 "spec": {
46 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } },
47 "template": {
48 "metadata": {
49 "labels": { "app": insertName },
50 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
51 },
52 "spec": {
53 "serviceAccountName": "victoriametrics",
54 "containers": [{
55 "name": "vminsert",
56 "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster",
57 "imagePullPolicy": "IfNotPresent",
58 "args": [
59 f"-http.pathPrefix={pipelineWebRoot}/vminsert/",
60 tzArg,
61 f"-storageNode={storageName}",
62 ],
63 "ports": [{ "containerPort": insertPort }]
64 }]
65 }
66 }
67 }
68 })) # yapf: disable
69
70
71 def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort):
72 (build / f'{storageFileName}_2deploy.yaml').write_text(
73 toJson({
74 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName },
75 "spec": {
76 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } },
77 "template": {
78 "metadata": {
79 "labels": { "app": storageName },
80 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/vmstorage/metrics", "prometheus.io/port": "80" }
81 },
82 "spec": {
83 "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }],
84 "serviceAccountName": "victoriametrics",
85 "containers": [{
86 "name": "vmstorage",
87 "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster",
88 "imagePullPolicy": "IfNotPresent",
89 "args": [
90 f"-http.pathPrefix={pipelineWebRoot}/vmstorage/",
91 tzArg,
92 f"-retentionPeriod={retention}",
93 f"-storageDataPath=/data/{pipelineName}",
94 ],
95 "ports": [
96 { "containerPort": 8482, "name": "http" },
97 { "containerPort": storageInsertPort, "name": "vminsert" },
98 { "containerPort": storageSelectPort, "name": "vmselect" },
99 ],
100 "volumeMounts": [{ "name": "data", "mountPath": "/data" }]
54 }], 101 }],
55 "volumeMounts": [{ 102 "affinity": affinityToNode(localPvHost)
56 "name": "config", 103 }
57 "mountPath": "/local/config" 104 }
105 }
106 })) # yapf: disable
107
108
109 def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort):
110 name = f"{objPrefix}-vmselect"
111 (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text(
112 toJson({
113 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
114 "spec": {
115 "replicas": 1,
116 "strategy": { "type": "Recreate" },
117 "selector": { "matchLabels": { "app": name } },
118 "template": {
119 "metadata": {
120 "labels": { "app": name },
121 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
122 },
123 "spec": {
124 "serviceAccountName": "victoriametrics",
125 "containers": [{
126 "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent",
127 "args": [
128 f"-http.pathPrefix={webRoot}/vmselect/",
129 tzArg,
130 ] + [f"-storageNode={n}" for n in storageSvcs],
131 "ports": [{ "containerPort": selectPort }]
58 }] 132 }]
59 }] 133 }
60 } 134 }
61 } 135 }
62 } 136 })) # yapf: disable
63 })) 137
64 (build / 'vmagent_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmagent"}, "spec": {"ports": [{"port": 80, "targetPort": 8429}], "selector": {"app": "next-victoriametrics-vmagent"}}})) 138 def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention):
65 139 agentName = f"{objPrefix}-{pipelineName}-vmagent"
66 (build / 'vminsert_deploy.yaml').write_text( 140 insertName = f"{objPrefix}-{pipelineName}-vminsert"
67 toJson({ 141 storageName = f"{objPrefix}-{pipelineName}-vmstorage"
68 "apiVersion": "apps/v1", 142
69 "kind": "Deployment", 143 agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent"
70 "metadata": { 144 insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert"
71 "name": "next-victoriametrics-vminsert" 145 storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage"
72 }, 146
73 "spec": { 147 localPvHost = "ditto"
74 "replicas": 1, 148 insertPort = 8480
75 "strategy": { 149 agentPort = 8429
76 "type": "Recreate" 150 storageInsertPort = 8400
77 }, 151 storageSelectPort = 8401
78 "selector": { 152 volName = f"{objPrefix}-data-{pipelineName}"
79 "matchLabels": { 153 request = "50Gi"
80 "app": "next-victoriametrics-vminsert" 154 pipelineWebRoot = f'{webRoot}/{pipelineName}'
81 } 155
82 }, 156 createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName)
83 "template": { 157 createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort)
84 "metadata": { 158 createPv(storageFileName, volName, request)
85 "labels": { 159 createPvc(storageFileName, volName, request)
86 "app": "next-victoriametrics-vminsert" 160 createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort)
87 }, 161
88 "annotations": { 162 createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}])
89 "prometheus.io/scrape": "true", 163 createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}])
90 "prometheus.io/path": "/m/metrics", 164 createSvc(storageFileName,storageName, [
91 "prometheus.io/port": "80" 165 {"port": 80, "targetPort": "http", "name": "http"},
92 } 166 {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"},
93 }, 167 {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"},
94 "spec": { 168 ]) # yapf: disable
95 "serviceAccountName": "victoriametrics", 169
96 "containers": [{ 170 return storageName
97 "name": "vminsert", 171
98 "image": "docker.io/victoriametrics/vminsert:v1.100.1-cluster", 172
99 "imagePullPolicy": "IfNotPresent", 173 def createIndex(objPrefix, webRoot, html):
100 "args": ["-http.pathPrefix=/m/next/vminsert/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"], 174 name = f'{objPrefix}-index'
101 "ports": [{ 175 httpServeRoot = '/opt/html'
102 "containerPort": 8480 176
103 }] 177 (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({
104 }] 178 "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name },
105 } 179 "data": {
106 } 180 "index.html": html,
107 } 181 (??) }
108 })) 182 (??) }))
109 (build / 'vminsert_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vminsert"}, "spec": {"ports": [{"port": 80, "targetPort": 8480}], "selector": {"app": "next-victoriametrics-vminsert"}}})) 183 (??)(build / 'vmselect_svc.yaml').write_text(toJson({
110 184 (??) "apiVersion": "v1",
111 (build / 'vmstorage_pv.yaml').write_text( 185 (??) "kind": "Service",
112 toJson({ 186 (??) "metadata": {
113 "apiVersion": "v1", 187 (??) "name": "next-victoriametrics-vmselect"
114 "kind": "PersistentVolume", 188 (??) },
115 "metadata": { 189 (??) "spec": {
116 "name": "next-victoriametrics", 190 (??) "ports": [{
117 "labels": { 191 (??) "port": 80,
118 "type": "local" 192 (??) "targetPort": 8481
119 } 193 (??) }],
120 }, 194 (??) "selector": {
121 "spec": { 195 (??) "app": "next-victoriametrics-vmselect"
122 "storageClassName": "manual", 196 (??) }
123 "hostPath": { 197 (??) }
124 "path": "/opt/next-victoriametrics" 198 (??)}
125 }, 199 (??) # in vmui, set server url to
126 "capacity": { 200 (??) # https://bigasterisk.com/m/next/vmselect/select/0/prometheus
127 "storage": "50Gi" 201 (??) ))
128 }, 202 (??)
129 "accessModes": ["ReadWriteMany"], 203 (??)(build / 'vmselect_svc.yaml').write_text(
130 "persistentVolumeReclaimPolicy": "Retain", 204 (??) toJson({
131 "claimRef": { 205 (??) "apiVersion": "networking.k8s.io/v1",
132 "namespace": "default", 206 (??) "kind": "Ingress",
133 "name": "next-victoriametrics" 207 (??) "metadata": {
134 } 208 (??) "name": "next-victoriametrics",
135 } 209 (??) "annotations": {
136 })) 210 (??) "cert-manager.io/cluster-issuer": "letsencrypt-prod",
137 (build / 'vmstorage_pvc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": "next-victoriametrics"}, "spec": {"storageClassName": "", "volumeName": "next-victoriametrics", "accessModes": ["ReadWriteMany"], "resources": {"requests": {"storage": "50Gi"}}}})) 211 (??) "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
138 212 (??) "ingress.pomerium.io/pass_identity_headers": "true",
139 (build / 'vmstorage_deploy.yaml').write_text( 213 (??) "ingress.pomerium.io/preserve_host_header": "true",
140 toJson({ 214 })) # yapf: disable
141 "apiVersion": "apps/v1", 215
142 "kind": "Deployment", 216 (build / f'{objPrefix}-3index_deploy.yaml').write_text(
143 "metadata": { 217 toJson({
144 "name": "next-victoriametrics-vmstorage" 218 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
145 }, 219 "spec": {
146 "spec": { 220 "replicas": 1,
147 "replicas": 1, 221 "selector": { "matchLabels": { "app": name } },
148 "strategy": { 222 "template": {
149 "type": "Recreate" 223 "metadata": { "labels": { "app": name } },
150 }, 224 "spec": {
151 "selector": { 225 "containers": [{
152 "matchLabels": { 226 "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent",
153 "app": "next-victoriametrics-vmstorage" 227 "args": [
154 } 228 f'--root={httpServeRoot}',
155 }, 229 '--directory-listing=true',
156 "template": { 230 '--experimental-metrics=true',
157 "metadata": { 231 ],
158 "labels": { 232 "ports": [{ "containerPort": 80 }],
159 "app": "next-victoriametrics-vmstorage" 233 "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }]
160 },
161 "annotations": {
162 "prometheus.io/scrape": "true",
163 "prometheus.io/path": "/m/vmstorage/metrics",
164 "prometheus.io/port": "80"
165 }
166 },
167 "spec": {
168 "volumes": [{
169 "name": "data",
170 "persistentVolumeClaim": {
171 "claimName": "next-victoriametrics"
172 }
173 }],
174 "serviceAccountName": "victoriametrics",
175 "containers": [{
176 "name": "vmstorage",
177 "image": "docker.io/victoriametrics/vmstorage:v1.100.1-cluster",
178 "imagePullPolicy": "IfNotPresent",
179 "args": ["-http.pathPrefix=/m/next/vmstorage/", "-loggerTimezone=America/Los_Angeles", "-retentionPeriod=100y", "-storageDataPath=/data/100y"],
180 "ports": [{
181 "containerPort": 8482,
182 "name": "http"
183 }, {
184 "containerPort": 8400,
185 "name": "vminsert"
186 }, {
187 "containerPort": 8401,
188 "name": "vmselect"
189 }], 234 }],
190 "volumeMounts": [{ 235 "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }]
191 "name": "data", 236 }
192 "mountPath": "/data" 237 }
193 }] 238 }
194 }], 239 })) # yapf: disable
195 "affinity": { 240 createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}])
196 "nodeAffinity": {
197 "requiredDuringSchedulingIgnoredDuringExecution": {
198 "nodeSelectorTerms": [{
199 "matchExpressions": [{
200 "key": "kubernetes.io/hostname",
201 "operator": "In",
202 "values": ["ditto"]
203 }]
204 }]
205 }
206 }
207 }
208 }
209 }
210 }
211 }))
212 (build / 'vmstorage_svc.yaml').write_text(toJson({"apiVersion": "v1", "kind": "Service", "metadata": {"name": "next-victoriametrics-vmstorage"}, "spec": {"ports": [{"port": 80, "targetPort": "http", "name": "http"}, {"port": 8400, "targetPort": "vminsert", "name": "vminsert"}, {"port": 8401, "targetPort": "vmselect", "name": "vmselect"}], "selector": {"app": "next-victoriametrics-vmstorage"}}}))
213
214 (build / 'vmselect_deploy.yaml').write_text(
215 toJson({
216 "apiVersion": "apps/v1",
217 "kind": "Deployment",
218 "metadata": {
219 "name": "next-victoriametrics-vmselect"
220 },
221 "spec": {
222 "replicas": 1,
223 "strategy": {
224 "type": "Recreate"
225 },
226 "selector": {
227 "matchLabels": {
228 "app": "next-victoriametrics-vmselect"
229 }
230 },
231 "template": {
232 "metadata": {
233 "labels": {
234 "app": "next-victoriametrics-vmselect"
235 },
236 "annotations": {
237 "prometheus.io/scrape": "true",
238 "prometheus.io/path": "/m/metrics",
239 "prometheus.io/port": "80"
240 }
241 },
242 "spec": {
243 "serviceAccountName": "victoriametrics",
244 "containers": [{
245 "name": "vmselect",
246 "image": "docker.io/victoriametrics/vmselect:v1.100.1-cluster",
247 "imagePullPolicy": "IfNotPresent",
248 "args": ["-http.pathPrefix=/m/next/vmselect/", "-loggerTimezone=America/Los_Angeles", "-storageNode=next-victoriametrics-vmstorage"],
249 "ports": [{
250 "containerPort": 8481
251 }]
252 }]
253 }
254 }
255 }
256 }))
257 (build / 'vmselect_svc.yaml').write_text(toJson({
258 "apiVersion": "v1",
259 "kind": "Service",
260 "metadata": {
261 "name": "next-victoriametrics-vmselect"
262 },
263 "spec": {
264 "ports": [{
265 "port": 80,
266 "targetPort": 8481
267 }],
268 "selector": {
269 "app": "next-victoriametrics-vmselect"
270 }
271 }
272 }
273 # in vmui, set server url to
274 # https://bigasterisk.com/m/next/vmselect/select/0/prometheus
275 ))
276 241
277 (build / 'vmselect_svc.yaml').write_text( 242 (build / 'vmselect_svc.yaml').write_text(
278 toJson({ 243 toJson({
279 "apiVersion": "networking.k8s.io/v1", 244 "apiVersion": "networking.k8s.io/v1",
280 "kind": "Ingress", 245 "kind": "Ingress",