view create_k8s.py @ 78:a4767069d8b0

alertmanager-conf is now a cmap
author drewp@bigasterisk.com
date Wed, 26 Jun 2024 22:29:10 -0700
parents 489c5d9a7dbc
children 4809a249d43f
line wrap: on
line source

from pathlib import Path
from index_page import makeIndexHtml
from output import affinityToNode, build, createIngress, createPv, createPvc, toJson, createSvc


def createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix):
    (build / f'{agentFileName}_deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": agentName },
            "spec": {
                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": agentName } },
                "template": {
                    "metadata": { "labels": { "app": agentName } },
                    "spec": {
                        "volumes": [{ "name": "config", "configMap": { "name": f"{objPrefix}-config" } }],
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "vmagent",
                            "image": f"docker.io/victoriametrics/vmagent:{vmVersion}",
                            "imagePullPolicy": "IfNotPresent",
                            "args": [
                                f"-http.pathPrefix={pipelineWebRoot}/vmagent/",
                                tzArg,
                                f"-promscrape.config=/local/config/{scrapeMapKey}",
                                "-promscrape.configCheckInterval=5s",
                                "-sortLabels",
                                f"-remoteWrite.url=http://{insertName}{pipelineWebRoot}/vminsert/insert/0/prometheus/api/v1/write",
                                "-remoteWrite.showURL",
                            ],
                            "ports": [{ "containerPort": agentPort }],
                            "volumeMounts": [{ "name": "config", "mountPath": "/local/config" }]
                        }]
                    }
                }
            }
        })) # yapf: disable


def createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort):
    (build / f'{insertFileName}_deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": insertName },
            "spec": {
                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": insertName } },
                "template": {
                    "metadata": { "labels": { "app": insertName } },
                    "spec": {
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "vminsert",
                            "image": f"docker.io/victoriametrics/vminsert:{vmVersion}-cluster",
                            "imagePullPolicy": "IfNotPresent",
                            "args": [
                                f"-http.pathPrefix={pipelineWebRoot}/vminsert/",
                                tzArg,
                                f"-storageNode={storageName}",
                            ],
                            "ports": [{ "containerPort": insertPort }]
                        }]
                    }
                }
            }
        })) # yapf: disable


def createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort):
    (build / f'{storageFileName}_2deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": storageName },
            "spec": {
                "replicas": 1, "strategy": { "type": "Recreate" }, "selector": { "matchLabels": { "app": storageName } },
                "template": {
                    "metadata": { "labels": { "app": storageName } },
                    "spec": {
                        "volumes": [{ "name": "data", "persistentVolumeClaim": { "claimName": volName } }],
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "vmstorage",
                            "image": f"docker.io/victoriametrics/vmstorage:{vmVersion}-cluster",
                            "imagePullPolicy": "IfNotPresent",
                            "args": [
                                f"-http.pathPrefix={pipelineWebRoot}/vmstorage/",
                                tzArg,
                                f"-retentionPeriod={retention}",
                                f"-storageDataPath=/data/{pipelineName}",
                            ],
                            "ports": [
                                { "containerPort": 8482, "name": "http" },
                                { "containerPort": storageInsertPort, "name": "vminsert" },
                                { "containerPort": storageSelectPort, "name": "vmselect" },
                            ],
                            "volumeMounts": [{ "name": "data", "mountPath": "/data" }]
                        }],
                        "affinity": affinityToNode(localPvHost)
                    }
                }
            }
        })) # yapf: disable


def createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort):
    name = f"{objPrefix}-vmselect"
    (build / f'{objPrefix}-1vmselect_deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
            "spec": {
                "replicas": 1,
                "strategy": { "type": "Recreate" },
                "selector": { "matchLabels": { "app": name } },
                "template": {
                    "metadata": { "labels": { "app": name } },
                    "spec": {
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "vmselect", "image": f"docker.io/victoriametrics/vmselect:{vmVersion}-cluster", "imagePullPolicy": "IfNotPresent",
                            "args": [
                                f"-http.pathPrefix={webRoot}/vmselect/",
                                tzArg,
                            ] + [f"-storageNode={n}" for n in storageSvcs],
                            "ports": [{ "containerPort": selectPort }]
                        }]
                    }
                }
            }
        })) # yapf: disable

def createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, pipelineName, scrapeMapKey, retention):
    agentName = f"{objPrefix}-{pipelineName}-vmagent"
    insertName = f"{objPrefix}-{pipelineName}-vminsert"
    storageName = f"{objPrefix}-{pipelineName}-vmstorage"

    agentFileName = f"{objPrefix}-0{pipelineName}-0vmagent"
    insertFileName = f"{objPrefix}-0{pipelineName}-1vminsert"
    storageFileName = f"{objPrefix}-0{pipelineName}-2vmstorage"

    localPvHost = "ditto"
    insertPort = 8480
    agentPort = 8429
    storageInsertPort = 8400
    storageSelectPort = 8401
    volName = f"{objPrefix}-data-{pipelineName}"
    request = "50Gi"
    pipelineWebRoot = f'{webRoot}/{pipelineName}'

    createAgentDeploy(tzArg, vmVersion, pipelineWebRoot, agentFileName, agentName, agentPort, scrapeMapKey, insertName, objPrefix)
    createInsertDeploy(tzArg, vmVersion, pipelineWebRoot, insertName, storageName, insertFileName, insertPort)
    createPv(storageFileName, volName, request)
    createPvc(storageFileName, volName, request)
    createStorageDeploy(tzArg, vmVersion, pipelineWebRoot, pipelineName, retention, storageName, storageFileName, localPvHost, volName, storageInsertPort, storageSelectPort)

    createSvc(agentFileName, agentName, [{"port": 80, "targetPort": agentPort}])
    createSvc(insertFileName, insertName, [{"port": 80, "targetPort": insertPort}])
    createSvc(storageFileName,storageName, [
        {"port": 80, "targetPort": "http", "name": "http"},
        {"port": storageInsertPort, "targetPort": "vminsert", "name": "vminsert"},
        {"port": storageSelectPort, "targetPort": "vmselect", "name": "vmselect"},
        ]) # yapf: disable

    return storageName


def createIndex(objPrefix, webRoot, html):
    name = f'{objPrefix}-index'
    httpServeRoot = '/opt/html'

    (build / f'{objPrefix}-3index_cmap.yaml').write_text(toJson({
        "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": name },
        "data": {
            "index.html": html,
            "index.js": Path("index.js").read_text(),
            "index.css": Path("index.css").read_text(),
        }
    })) # yapf: disable

    (build / f'{objPrefix}-3index_deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": name },
            "spec": {
                "replicas": 1,
                "selector": { "matchLabels": { "app": name } },
                "template": {
                    "metadata": { "labels": { "app": name } },
                    "spec": {
                        "containers": [{
                            "name": "webserver", "image": "docker.io/joseluisq/static-web-server", "imagePullPolicy": "IfNotPresent",
                            "args": [
                                f'--root={httpServeRoot}',
                                '--directory-listing=true',
                                '--experimental-metrics=true',
                            ],
                            "ports": [{ "containerPort": 80 }],
                            "volumeMounts": [{ "name": "html", "mountPath": f"{httpServeRoot}{webRoot}" }]
                        }],
                        "volumes": [{ "name": "html", "configMap": { "name": name, "defaultMode": 444 } }]
                    }
                }
            }
        })) # yapf: disable
    createSvc(f'{objPrefix}-3index', f'{objPrefix}-index', [{'port': 80, 'targetPort': 80}])


def createVmalertObjs(tzArg, objPrefix, webRoot, policy, vmVersion):
    vmalertName = f"{objPrefix}-vmalert"
    (build / f'{objPrefix}-4vmalert_2deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": vmalertName },
            "spec": {
                "replicas": 1,
                "strategy": { "type": "Recreate" },
                "selector": { "matchLabels": { "app": vmalertName } },
                "template": {
                    "metadata": { "labels": { "app": vmalertName }, },
                    "spec": {
                        "volumes": [{ "name": "config", "configMap": { "name": "victoriametrics-config" } }],
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "vmalert", "image": f"docker.io/victoriametrics/vmalert:{vmVersion}", "imagePullPolicy": "IfNotPresent",
                            "args": [
                                tzArg,
                                "-loggerLevel=INFO",
                                "-configCheckInterval=5s",
                                "-datasource.queryStep=5m",
                                "-evaluationInterval=1m",
                                "-rule=/local/rules",

                                # used in a field of the alert to link back to vmalert's info page
                                f"-external.url=https://bigasterisk.com{webRoot}/vmalert",

                                f"-datasource.url=http://{objPrefix}-vmselect/m/vmselect/select/0/prometheus",
                                f"-http.pathPrefix={webRoot}/vmalert/",
                                f"-notifier.url=http://{objPrefix}-alertmanager/m/alertmanager",

                                # todo: remoteWrite should go to vminsert; not sure how to test it
                                # "-remoteRead.url=http://victoriametrics/m/",
                                # "-remoteWrite.url=http://victoriametrics/m/",
                            ],
                            "ports": [{ "containerPort": 8880 }],
                            "volumeMounts": [{ "name": "config", "mountPath": "/local" }]
                        }],
                    }
                }
            }
        }))  # yapf: disable

    createSvc(f'{objPrefix}-4vmalert', vmalertName, [{"port": 80, "targetPort": 8880}])

    createIngress(
        f'{objPrefix}-4vmalert_ingress.yaml',
        'vmalert',
        policy,
        [{ "pathType": "Prefix", "path": f"{webRoot}/vmalert/", "backend": { "service": { "name": vmalertName, "port": { "number": 80 } } } }],
        "bigasterisk.com",
    )  # yapf: disable


def createAlertmanagerObjs(tzArg, objPrefix, webRoot, policy):
    confFilename = "alertmanager.yml"

    (build / f'{objPrefix}-5alertmanager_conf_cmap.yaml').write_text(toJson({
        "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": "alertmanager-conf" },
        "data": {
            confFilename: Path("./alertmanager.yaml").read_text(),
        }
    })) # yapf: disable
    createPv(f'{objPrefix}-5alertmanager', "opt-alertmanager", '50Gi')
    createPvc(f'{objPrefix}-5alertmanager', "opt-alertmanager", '50Gi')

    amName = f'{objPrefix}-alertmanager'
    (build / f'{objPrefix}-5alertmanager_deploy.yaml').write_text(
        toJson({
            "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": amName },
            "spec": {
                "replicas": 1,
                "selector": { "matchLabels": { "app": amName } },
                "template": {
                    "metadata": { "labels": { "app": amName } },
                    "spec": {
                        "volumes": [
                            { "name": "opt-alertmanager", "persistentVolumeClaim": { "claimName": "opt-alertmanager" } },
                            { "name": "alertmanager-conf", "configMap": { "name": "alertmanager-conf" } }
                        ],
                        "serviceAccountName": "victoriametrics",
                        "containers": [{
                            "name": "alertmanager", "image": "docker.io/prom/alertmanager:v0.27.0", "imagePullPolicy": "IfNotPresent",
                            "args": [
                                "--log.level=info",
                                f"--config.file=/conf/{confFilename}",
                                f"--web.external-url=https://bigasterisk.com{webRoot}/alertmanager/",
                                f"--web.route-prefix={webRoot}/alertmanager",
                            ],
                            "ports": [{ "containerPort": 9093 }],
                            "volumeMounts": [
                                { "name": "opt-alertmanager", "mountPath": "/alertmanager" },
                                { "name": "alertmanager-conf", "mountPath": "/conf" }
                            ]
                        }],
                        "affinity": affinityToNode("ditto"),
                    }
                }
            }
        }))  # yapf: disable

    createSvc(f'{objPrefix}-5alertmanager', amName, [{"port": 80, "targetPort": 9093}])
    createIngress(
        f'{objPrefix}-5alertmanager_ingress.yaml',
        amName,
        policy,
        [{ "pathType": "Prefix", "path": "/m/alertmanager/", "backend": { "service": { "name": amName, "port": { "number": 80 } } } }],
        "bigasterisk.com",
    )  # yapf: disable


def main():
    tzArg = "-loggerTimezone=America/Los_Angeles"
    objPrefix = "victoriametrics"  # prefix on all k8s object names
    webRoot = "/m"
    vmVersion = "v1.100.1"
    webHost = 'bigasterisk.com'
    pipelines = [
        ('forever', '100y'),
        ('recent', '90y'),
    ]
    storageSvcs = [createIngestPipeline(tzArg, vmVersion, webRoot, objPrefix, p, f'scrape_{p}.yaml', ret) for p, ret in pipelines]

    selectPort = 8481
    createVmselectDeploy(tzArg, vmVersion, webRoot, objPrefix, storageSvcs, selectPort)
    createSvc(f'{objPrefix}-1vmselect', f"{objPrefix}-vmselect", [{"port": 80, "targetPort": selectPort}])

    ingressPaths = [
        { "pathType": "Prefix", "path": f"{webRoot}/",          "backend": { "service": { "name": f"{objPrefix}-index",    "port": { "number": 80 } } } },
        { "pathType": "Prefix", "path": f"{webRoot}/vmselect/", "backend": { "service": { "name": f"{objPrefix}-vmselect", "port": { "number": 80 } } } },
    ]  # yapf: disable
    for p, _ in pipelines:
        ingressPaths.extend([
            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vmagent/",   "backend": { "service": { "name": f"{objPrefix}-{p}-vmagent",   "port": { "number": 80 } } } },
            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vminsert/",  "backend": { "service": { "name": f"{objPrefix}-{p}-vminsert",  "port": { "number": 80 } } } },
            { "pathType": "Prefix", "path": f"{webRoot}/{p}/vmstorage/", "backend": { "service": { "name": f"{objPrefix}-{p}-vmstorage", "port": { "number": 80 } } } },
        ]) # yapf: disable

    policy = """\
allow:
    or:
        - { email: { is: "drewpca@gmail.com" }}
        - { email: { is: "kelsimp@gmail.com" }}
    """
    createIngress(f'{objPrefix}-2ingress.yaml', objPrefix, policy, ingressPaths, webHost)
    createIndex(objPrefix, webRoot, makeIndexHtml(objPrefix, webRoot, webHost))
    createVmalertObjs(tzArg, objPrefix, webRoot, policy, vmVersion)
    createAlertmanagerObjs(tzArg, objPrefix, webRoot, policy)


main()