|
56
|
1 from index_page import makeIndexHtml
|
|
|
2 from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc
|
|
55
|
3
|
|
|
4
|
|
56
|
5 def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName):
|
|
|
6 (build / f'{agentFileName}_deploy.yaml').write_text(
|
|
|
7 toJson({
|
|
|
8 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName },
|
|
|
9 "spec": {
|
|
|
10 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } },
|
|
|
11 "template": {
|
|
|
12 "metadata": {
|
|
|
13 "labels": { "app": agentName },
|
|
|
14 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": f"{pipelineWebRoot}/vmagent/metrics", "prometheus.io/port": "80" }
|
|
55
|
15 },
|
|
56
|
16 "spec": {
|
|
|
17 "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }],
|
|
|
18 "serviceAccountName": "victoriametrics",
|
|
|
19 "containers": [{
|
|
|
20 "name": "vmagent",
|
|
|
21 "image": f"docker.io/victoriametrics/vmagent:{vmVersion}",
|
|
|
22 "imagePullPolicy": "IfNotPresent",
|
|
|
23 "args": [
|
|
|
24 f"-http.pathPrefix={pipelineWebRoot}/vmagent/",
|
|
|
25 tzArg,
|
|
|
26 f"-promscrape.config=/local/config/{scrapeMapKey}",
|
|
|
27 "-promscrape.configCheckInterval=5s",
|
|
|
28 "-sortLabels",
|
|
|
29 f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write",
|
|
|
30 "-remoteWrite.showURL",
|
|
|
31 ],
|
|
|
32 "ports": [{ "containerPort": agentPort }],
|
|
|
33 "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }]
|
|
|
34 }]
|
|
55
|
35 }
|
|
|
36 }
|
|
|
37 }
|
|
56
|
38 })) # yapf: disable
|
|
|
39
|
|
55
|
40
|
|
56
|
41 def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort):
|
|
|
42 (build / f'{insertFileName}_deploy.yaml').write_text(
|
|
|
43 toJson({
|
|
|
44 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName },
|
|
|
45 "spec": {
|
|
|
46 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } },
|
|
|
47 "template": {
|
|
|
48 "metadata": {
|
|
|
49 "labels": { "app": insertName },
|
|
|
50 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
|
|
55
|
51 },
|
|
56
|
52 "spec": {
|
|
|
53 "serviceAccountName": "victoriametrics",
|
|
|
54 "containers": [{
|
|
|
55 "name": "vminsert",
|
|
|
56 "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster",
|
|
|
57 "imagePullPolicy": "IfNotPresent",
|
|
|
58 "args": [
|
|
|
59 f"-http.pathPrefix={pipelineWebRoot}/vminsert/",
|
|
|
60 tzArg,
|
|
|
61 f"-storageNode={storageName}",
|
|
|
62 ],
|
|
|
63 "ports": [{ "containerPort": insertPort }]
|
|
|
64 }]
|
|
55
|
65 }
|
|
|
66 }
|
|
|
67 }
|
|
56
|
68 })) # yapf: disable
|
|
55
|
69
|
|
|
70
|
|
56
|
71 def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort):
|
|
|
72 (build / f'{storageFileName}_2deploy.yaml').write_text(
|
|
|
73 toJson({
|
|
|
74 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName },
|
|
|
75 "spec": {
|
|
|
76 "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } },
|
|
|
77 "template": {
|
|
|
78 "metadata": {
|
|
|
79 "labels": { "app": storageName },
|
|
|
80 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/vmstorage/metrics", "prometheus.io/port": "80" }
|
|
55
|
81 },
|
|
56
|
82 "spec": {
|
|
|
83 "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }],
|
|
|
84 "serviceAccountName": "victoriametrics",
|
|
|
85 "containers": [{
|
|
|
86 "name": "vmstorage",
|
|
|
87 "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster",
|
|
|
88 "imagePullPolicy": "IfNotPresent",
|
|
|
89 "args": [
|
|
|
90 f"-http.pathPrefix={pipelineWebRoot}/vmstorage/",
|
|
|
91 tzArg,
|
|
|
92 f"-retentionPeriod={retention}",
|
|
|
93 f"-storageDataPath=/data/{pipelineName}",
|
|
|
94 ],
|
|
|
95 "ports": [
|
|
|
96 { "containerPort": 8482, "name": "http" },
|
|
|
97 { "containerPort": storageInsertPort, "name": "vminsert" },
|
|
|
98 { "containerPort": storageSelectPort, "name": "vmselect" },
|
|
|
99 ],
|
|
|
100 "volumeMounts": [{ "name": "data", "mountPath": "/data" }]
|
|
55
|
101 }],
|
|
56
|
102 "affinity": affinityToNode(localPvHost)
|
|
55
|
103 }
|
|
|
104 }
|
|
|
105 }
|
|
56
|
106 })) # yapf: disable
|
|
|
107
|
|
55
|
108
|
|
56
|
109 def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort):
|
|
|
110 name = f"{objPrefix}-vmselect"
|
|
|
111 (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text(
|
|
|
112 toJson({
|
|
|
113 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
|
|
|
114 "spec": {
|
|
|
115 "replicas": 1,
|
|
|
116 "strategy": { "type": "Recreate" },
|
|
|
117 "selector": { "matchLabels": { "app": name } },
|
|
|
118 "template": {
|
|
|
119 "metadata": {
|
|
|
120 "labels": { "app": name },
|
|
|
121 "annotations": { "prometheus.io/scrape": "true", "prometheus.io/path": "/m/metrics", "prometheus.io/port": "80" }
|
|
55
|
122 },
|
|
56
|
123 "spec": {
|
|
|
124 "serviceAccountName": "victoriametrics",
|
|
|
125 "containers": [{
|
|
|
126 "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent",
|
|
|
127 "args": [
|
|
|
128 f"-http.pathPrefix={webRoot}/vmselect/",
|
|
|
129 tzArg,
|
|
|
130 ] + [f"-storageNode={n}" for n in storageSvcs],
|
|
|
131 "ports": [{ "containerPort": selectPort }]
|
|
|
132 }]
|
|
55
|
133 }
|
|
|
134 }
|
|
|
135 }
|
|
56
|
136 })) # yapf: disable
|
|
|
137
|
|
|
138 def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention):
|
|
|
139 agentName = f"{objPrefix}-{pipelineName}-vmagent"
|
|
|
140 insertName = f"{objPrefix}-{pipelineName}-vminsert"
|
|
|
141 storageName = f"{objPrefix}-{pipelineName}-vmstorage"
|
|
|
142
|
|
|
143 agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent"
|
|
|
144 insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert"
|
|
|
145 storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage"
|
|
|
146
|
|
|
147 localPvHost = "ditto"
|
|
|
148 insertPort = 8480
|
|
|
149 agentPort = 8429
|
|
|
150 storageInsertPort = 8400
|
|
|
151 storageSelectPort = 8401
|
|
|
152 volName = f"{objPrefix}-data-{pipelineName}"
|
|
|
153 request = "50Gi"
|
|
|
154 pipelineWebRoot = f'{webRoot}/{pipelineName}'
|
|
|
155
|
|
|
156 createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, pipelineName)
|
|
|
157 createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort)
|
|
|
158 createPv(storageFileName, volName, request)
|
|
|
159 createPvc(storageFileName, volName, request)
|
|
|
160 createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort)
|
|
|
161
|
|
|
162 createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}])
|
|
|
163 createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}])
|
|
|
164 createSvc(storageFileName,storageName, [
|
|
|
165 {"port": 80, "targetPort": "http", "name": "http"},
|
|
|
166 {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"},
|
|
|
167 {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"},
|
|
|
168 ]) # yapf: disable
|
|
|
169
|
|
|
170 return storageName
|
|
|
171
|
|
|
172
|
|
|
173 def createIndex(objPrefix, webRoot, html):
|
|
|
174 name = f'{objPrefix}-index'
|
|
|
175 httpServeRoot = '/opt/html'
|
|
|
176
|
|
|
177 (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({
|
|
|
178 "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name },
|
|
|
179 "data": {
|
|
|
180 "index.html": html,
|
|
|
181 (??) }
|
|
|
182 (??) }))
|
|
|
183 (??)(build / 'vmselect_svc.yaml').write_text(toJson({
|
|
|
184 (??) "apiVersion": "v1",
|
|
|
185 (??) "kind": "Service",
|
|
|
186 (??) "metadata": {
|
|
|
187 (??) "name": "next-victoriametrics-vmselect"
|
|
|
188 (??) },
|
|
|
189 (??) "spec": {
|
|
|
190 (??) "ports": [{
|
|
|
191 (??) "port": 80,
|
|
|
192 (??) "targetPort": 8481
|
|
|
193 (??) }],
|
|
|
194 (??) "selector": {
|
|
|
195 (??) "app": "next-victoriametrics-vmselect"
|
|
|
196 (??) }
|
|
|
197 (??) }
|
|
|
198 (??)}
|
|
|
199 (??) # in vmui, set server url to
|
|
|
200 (??) # https://bigasterisk.com/m/next/vmselect/select/0/prometheus
|
|
|
201 (??) ))
|
|
|
202 (??)
|
|
|
203 (??)(build / 'vmselect_svc.yaml').write_text(
|
|
|
204 (??) toJson({
|
|
|
205 (??) "apiVersion": "networking.k8s.io/v1",
|
|
|
206 (??) "kind": "Ingress",
|
|
|
207 (??) "metadata": {
|
|
|
208 (??) "name": "next-victoriametrics",
|
|
|
209 (??) "annotations": {
|
|
|
210 (??) "cert-manager.io/cluster-issuer": "letsencrypt-prod",
|
|
|
211 (??) "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
|
|
|
212 (??) "ingress.pomerium.io/pass_identity_headers": "true",
|
|
|
213 (??) "ingress.pomerium.io/preserve_host_header": "true",
|
|
|
214 })) # yapf: disable
|
|
|
215
|
|
|
216 (build / f'{objPrefix}-3index_deploy.yaml').write_text(
|
|
|
217 toJson({
|
|
|
218 "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
|
|
|
219 "spec": {
|
|
|
220 "replicas": 1,
|
|
|
221 "selector": { "matchLabels": { "app": name } },
|
|
|
222 "template": {
|
|
|
223 "metadata": { "labels": { "app": name } },
|
|
|
224 "spec": {
|
|
|
225 "containers": [{
|
|
|
226 "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent",
|
|
|
227 "args": [
|
|
|
228 f'--root={httpServeRoot}',
|
|
|
229 '--directory-listing=true',
|
|
|
230 '--experimental-metrics=true',
|
|
|
231 ],
|
|
|
232 "ports": [{ "containerPort": 80 }],
|
|
|
233 "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }]
|
|
|
234 }],
|
|
|
235 "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }]
|
|
|
236 }
|
|
|
237 }
|
|
|
238 }
|
|
|
239 })) # yapf: disable
|
|
|
240 createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}])
|
|
55
|
241
|
|
|
242 (build / 'vmselect_svc.yaml').write_text(
|
|
|
243 toJson({
|
|
|
244 "apiVersion": "networking.k8s.io/v1",
|
|
|
245 "kind": "Ingress",
|
|
|
246 "metadata": {
|
|
|
247 "name": "next-victoriametrics",
|
|
|
248 "annotations": {
|
|
|
249 "cert-manager.io/cluster-issuer": "letsencrypt-prod",
|
|
|
250 "ingress.pomerium.io/allow_public_unauthenticated_access": "false",
|
|
|
251 "ingress.pomerium.io/pass_identity_headers": "true",
|
|
|
252 "ingress.pomerium.io/preserve_host_header": "true",
|
|
|
253 }
|
|
|
254 },
|
|
|
255 "spec": {
|
|
|
256 "ingressClassName": "pomerium",
|
|
|
257 "rules": [{
|
|
|
258 "host": "bigasterisk.com",
|
|
|
259 "http": {
|
|
|
260 "paths": [{
|
|
|
261 "pathType": "Prefix",
|
|
|
262 "path": "/m/next/vmselect/",
|
|
|
263 "backend": {
|
|
|
264 "service": {
|
|
|
265 "name": "next-victoriametrics-vmselect",
|
|
|
266 "port": {
|
|
|
267 "number": 80
|
|
|
268 }
|
|
|
269 }
|
|
|
270 }
|
|
|
271 }, {
|
|
|
272 "pathType": "Prefix",
|
|
|
273 "path": "/m/next/vmagent/",
|
|
|
274 "backend": {
|
|
|
275 "service": {
|
|
|
276 "name": "next-victoriametrics-vmagent",
|
|
|
277 "port": {
|
|
|
278 "number": 80
|
|
|
279 }
|
|
|
280 }
|
|
|
281 }
|
|
|
282 }, {
|
|
|
283 "pathType": "Prefix",
|
|
|
284 "path": "/m/next/vmselect/",
|
|
|
285 "backend": {
|
|
|
286 "service": {
|
|
|
287 "name": "next-victoriametrics-vmselect",
|
|
|
288 "port": {
|
|
|
289 "number": 80
|
|
|
290 }
|
|
|
291 }
|
|
|
292 }
|
|
|
293 }]
|
|
|
294 }
|
|
|
295 }],
|
|
|
296 "tls": [{
|
|
|
297 "hosts": ["bigasterisk.com"],
|
|
|
298 "secretName": "bigasterisk.com-tls"
|
|
|
299 }]
|
|
|
300 }
|
|
|
301 }))
|
