view tasks.py @ 65:fada8d64c4d3

notes
author drewp@bigasterisk.com
date Thu, 02 May 2024 23:15:37 -0700
parents 84a4c4cca4a5
children
line wrap: on
line source

from pathlib import Path
import yaml
from invoke import task
from kubernetes import config

import alert_rules

from k8s_ops import firstPodName, refreshPodCmaps, replaceCmap

config.load_kube_config()


@task
def push_config(ctx):
    rulesObj = alert_rules.allRules(ctx)

    replaceCmap("victoriametrics-config", {
        "scrape_main": scrapeConfig("config/scrape_main.yaml"),
        "scrape_recent": scrapeConfig("config/scrape_recent.yaml"),
        "scrape_forever": scrapeConfig("config/scrape_forever.yaml"),
        "rules": rulesObj,
    })

    # these don't give errors on rules format! they just quietly keep the old
    # rules! use `skaffold run` to get errs.
    #
    # or run
    #  validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
    #  validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")

    refreshPodCmaps(firstPodName("app=victoriametrics"))
    refreshPodCmaps(firstPodName("app=vmalert"))

    # If the VM reloader isn't fast enough, we could do this too:
    # hup(ctx, 'deploy/victoriametrics', 'victoria-metrics-prod')


@task
def push_config_2024(ctx):
    # plan:
    #   every discovered service may:
    #      - be described here as a forever retention - ignore the discovery
    #      - be blocked here as a no-metrics service - ignore the discovery
    #      - be scraped as 'recent', with possible overrides of port/path
    #   all per-node metrics shall be 'recent' (oops, not smartctl!)
    map: dict[str, object] = {
        'rules': alert_rules.allRules(ctx),
    }
    top=Path('config/build/scrape_jobs')
    for p in top.glob('**/*.yaml'):
        map[str(p.relative_to(top))] = scrapeConfig(p)
    replaceCmap("next-victoriametrics-config", map)
    refreshPodCmaps(firstPodName("app=next-victoriametrics-forever-vmagent"))
    refreshPodCmaps(firstPodName("app=next-victoriametrics-recent-vmagent"))
  

def scrapeConfig(fn):
    return yaml.load(open(fn), yaml.FullLoader)