Mercurial > code > home > repos > victoriametrics
changeset 72:85d9dae18656
fix up vmalert k8s objs
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 03 May 2024 13:33:33 -0700 |
| parents | bfacf01fd119 |
| children | c3d13fab248e |
| files | create_k8s.py create_scrape_configs.py index_page.py |
| diffstat | 3 files changed, 75 insertions(+), 215 deletions(-) [+] |
line wrap: on
line diff
--- a/create_k8s.py Fri May 03 12:27:16 2024 -0700 +++ b/create_k8s.py Fri May 03 13:33:33 2024 -0700 @@ -211,131 +211,64 @@ createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}]) -def createAlertObjs(objPrefix, webRoot): - (build / f'{objPrefix}-4vmalert_0pv.yaml').write_text( - toJson({ - "apiVersion": "v1", - "kind": "PersistentVolume", - "metadata": { - "name": "opt-alertmanager", - "labels": { - "type": "local" - } - }, - "spec": { - "storageClassName": "manual", - "hostPath": { - "path": "/opt/alertmanager" - }, - "capacity": { - "storage": "50Gi" - }, - "accessModes": ["ReadWriteOnce"], - "persistentVolumeReclaimPolicy": "Retain", - "claimRef": { - "namespace": "default", - "name": "opt-alertmanager" - } - } - })) - (build / f'{objPrefix}-4vmalert_1pvc.yaml').write_text(toJson({ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": { - "name": "opt-alertmanager" - }, - "spec": { - "storageClassName": "", - "volumeName": "opt-alertmanager", - "accessModes": ["ReadWriteOnce"], - "resources": { - "requests": { - "storage": "50Gi" - } - } - }, - })) +def createVmalertObjs(tzArg, objPrefix, webRoot, policy, vmVersion): + vmalertName = f"{objPrefix}-vmalert" (build / f'{objPrefix}-4vmalert_2deploy.yaml').write_text( toJson({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "vmalert" - }, + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": vmalertName }, "spec": { "replicas": 1, - "strategy": { - "type": "Recreate" - }, - "selector": { - "matchLabels": { - "app": "vmalert" - } - }, + "strategy": { "type": "Recreate" }, + "selector": { "matchLabels": { "app": vmalertName } }, "template": { - "metadata": { - "labels": { - "app": "vmalert" - }, - "annotations": { - "prometheus.io/scrape": "true" - } - }, + "metadata": { "labels": { "app": vmalertName }, }, "spec": { - "volumes": [{ - "name": "config", - "configMap": { - "name": "victoriametrics-config" - } - }], - "serviceAccountName": - "victoriametrics", + "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }], + "serviceAccountName": "victoriametrics", "containers": [{ - "name": "vmalert", - "image": "docker.io/victoriametrics/vmalert:v1.91.2", + "name": "vmalert", "image": f"docker.io/victoriametrics/vmalert:{vmVersion}", "imagePullPolicy": "IfNotPresent", "args": [ + tzArg, + "-loggerLevel=INFO", "-configCheckInterval=5s", - "-datasource.url=http://victoriametrics/m/", "-datasource.queryStep=5m", "-evaluationInterval=1m", - "-external.url=https://bigasterisk.com/vmalert", - "-loggerLevel=INFO", - "-loggerTimezone=America/Los_Angeles", - "-memory.allowedBytes=512MB", - "-notifier.url=http://alertmanager", - "-remoteRead.url=http://victoriametrics/m/", - "-remoteWrite.url=http://victoriametrics/m/", "-rule=/local/rules", + + # used in a field of the alert to link back to vmalert's info page + f"-external.url=https://bigasterisk.com{webRoot}/vmalert", + + f"-datasource.url=http://{objPrefix}-vmselect/m/vmselect/select/0/prometheus", + f"-http.pathPrefix={webRoot}/vmalert/", + f"-notifier.url=http://{objPrefix}-alertmanager", + + # todo: remoteWrite should go to vminsert; not sure how to test it + # "-remoteRead.url=http://victoriametrics/m/", + # "-remoteWrite.url=http://victoriametrics/m/", ], - "ports": [{ - "containerPort": 8880 - }], - "volumeMounts": [{ - "name": "config", - "mountPath": "/local" - }] - }] + "ports": [{ "containerPort": 8880 }], + "volumeMounts": [{ "name": "config", "mountPath": "/local" }] + }], } } } - })) + })) # yapf: disable + + createSvc(f'{objPrefix}-4vmalert', vmalertName, [{"port": 80, "targetPort": 8880}]) - (build / f'{objPrefix}-4vmalert_svc.yaml').write_text(toJson({ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "vmalert" - }, - "spec": { - "ports": [{ - "port": 80, - "targetPort": 8880 - }], - "selector": { - "app": "vmalert" - } - }, - })) + createIngress( + f'{objPrefix}-4vmalert_ingress.yaml', + 'vmalert', + policy, + [{ "pathType": "Prefix", "path": f"{webRoot}/vmalert/", "backend": { "service": { "name": vmalertName, "port": { "number": 80 } } } }], + "bigasterisk.com", + ) # yapf: disable + + +def createAlertmanagerObjs(tzArg, objPrefix, webRoot, policy): + createPv(f'{objPrefix}-4vmalert', "opt-alertmanager", '50Gi') + createPvc(f'{objPrefix}-4vmalert', "opt-alertmanager", '50Gi') + (build / f'{objPrefix}-5alertmanager_deploy.yaml').write_text( toJson({ "apiVersion": "apps/v1", @@ -367,6 +300,7 @@ "containers": [{ "name": "alertmanager", "image": "docker.io/prom/alertmanager:v0.27.0", + "imagePullPolicy": "IfNotPresent", "args": [ "--config.file=/alertmanager/alertmanager.yml", "--web.external-url=https://bigasterisk.com/alertmanager/", @@ -381,118 +315,32 @@ "mountPath": "/alertmanager" }] }], - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kubernetes.io/hostname", - "operator": "In", - "values": ["ditto"] - }] - }] - } - } - } + "affinity": affinityToNode("ditto"), } } } })) - (build / f'{objPrefix}-5alertmanager_svc.yaml').write_text(toJson({ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "alertmanager" - }, - "spec": { - "ports": [{ - "port": 80, - "targetPort": 9093 - }], - "selector": { - "app": "alertmanager" - } - }, - })) - (build / f'{objPrefix}-4vmalert_ingress.yaml').write_text( - toJson({ - "apiVersion": "networking.k8s.io/v1", - "kind": "Ingress", - "metadata": { - "name": "vmalert", - "annotations": { - "cert-manager.io/cluster-issuer": "letsencrypt-prod", - "ingress.pomerium.io/allow_public_unauthenticated_access": "false", - "ingress.pomerium.io/pass_identity_headers": "true", - "ingress.pomerium.io/preserve_host_header": "true", - "ingress.pomerium.io/policy": "allow:\n or: \n - { email: { is: \"drewpca@gmail.com\" }}\n - { email: { is: \"kelsimp@gmail.com\" }}\n" - } - }, - "spec": { - "ingressClassName": "pomerium", - "rules": [{ - "host": "bigasterisk.com", - "http": { - "paths": [{ - "pathType": "Prefix", - "path": "/vmalert/", - "backend": { - "service": { - "name": "vmalert", - "port": { - "number": 80 - } - } - } - }] + + createSvc(f'{objPrefix}-5alertmanager', "alertmanager", [{"port": 80, "targetPort": 9093}]) + createIngress( + f'{objPrefix}-5alertmanager_ingress.yaml', + "alertmanager", + policy, + # "ingress.pomerium.io/prefix_rewrite": "/" + [{ + "pathType": "Prefix", + "path": "/alertmanager/", + "backend": { + "service": { + "name": "alertmanager", + "port": { + "number": 80 } - }], - "tls": [{ - "hosts": ["bigasterisk.com"], - "secretName": "bigasterisk.com-tls" - }] + } } - })) - (build / f'{objPrefix}-5alertmanager_ingress.yaml').write_text( - toJson({ - "apiVersion": "networking.k8s.io/v1", - "kind": "Ingress", - "metadata": { - "name": "alertmanager", - "annotations": { - "cert-manager.io/cluster-issuer": "letsencrypt-prod", - "ingress.pomerium.io/allow_public_unauthenticated_access": "false", - "ingress.pomerium.io/pass_identity_headers": "true", - "ingress.pomerium.io/preserve_host_header": "true", - "ingress.pomerium.io/policy": "allow:\n or: \n - { email: { is: \"drewpca@gmail.com\" }}\n - { email: { is: \"kelsimp@gmail.com\" }}\n", - "ingress.pomerium.io/prefix_rewrite": "/" - } - }, - "spec": { - "ingressClassName": "pomerium", - "rules": [{ - "host": "bigasterisk.com", - "http": { - "paths": [{ - "pathType": "Prefix", - "path": "/alertmanager/", - "backend": { - "service": { - "name": "alertmanager", - "port": { - "number": 80 - } - } - } - }] - } - }], - "tls": [{ - "hosts": ["bigasterisk.com"], - "secretName": "bigasterisk.com-tls" - }] - } - })) + }], + "bigasterisk.com", + ) def main(): @@ -530,7 +378,8 @@ """ createIngress(f'{objPrefix}-2ingress.yaml', objPrefix, policy, ingressPaths, webHost) createIndex(objPrefix, webRoot, makeIndexHtml(objPrefix, webRoot, webHost)) - createAlertObjs(objPrefix, webRoot) + createVmalertObjs(tzArg, objPrefix, webRoot, policy, vmVersion) + # createAlertmanagerObjs(tzArg, objPrefix, webRoot, policy) main()
--- a/create_scrape_configs.py Fri May 03 12:27:16 2024 -0700 +++ b/create_scrape_configs.py Fri May 03 13:33:33 2024 -0700 @@ -61,6 +61,7 @@ jobConfig(name="victoriametrics-recent-vminsert", metrics_path='/m/recent/vminsert/metrics', targets=FromName), jobConfig(name="victoriametrics-recent-vmstorage", metrics_path='/m/recent/vmstorage/metrics', targets=FromName), jobConfig(name="victoriametrics-vmselect", metrics_path='/m/vmselect/metrics', targets=FromName), + jobConfig(name="victoriametrics-vmalert", metrics_path='/m/vmalert/metrics', targets=FromName), jobConfig(name="victoriametrics-index", targets=FromName), # todo:
--- a/index_page.py Fri May 03 12:27:16 2024 -0700 +++ b/index_page.py Fri May 03 13:33:33 2024 -0700 @@ -67,6 +67,16 @@ </table> </section> + <section> + <h2>vmalert</h2> + <table> + <tr> + <td><a href="/vmalert">vmalert</a></td> + <td><a href="/vmalert/metrics">metrics</a></td> + </tr> + </table> + </section> + <script src="index.js"></script> <script> init("https://{webHost}{webRoot}/vmselect/select/0/prometheus");