# HG changeset patch # User drewp@bigasterisk.com # Date 1707966649 28800 # Node ID f05be84926e2793d8c04ecd543363aeec778f19f # Parent 2f87ecd2a754e8ab8636a9bed387f22594f2e4f1 drop this kubelet rule because kubelet_container_log_filesystem_used_bytes doesn't seem to have 'instance' anymore diff -r 2f87ecd2a754 -r f05be84926e2 alert_rules.py --- a/alert_rules.py Tue Jan 23 14:56:51 2024 -0800 +++ b/alert_rules.py Wed Feb 14 19:10:49 2024 -0800 @@ -119,7 +119,6 @@ "interval": "1m", "rules": k8sRules(), }, - expectedK8sNodesGroup(ctx), # # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name { @@ -366,19 +365,3 @@ return _runJson(ctx, "cd /my/serv/lanscape; pdm run python hosts_expected_online.py") -def expectedK8sNodesGroup(ctx): - getNode = _runJson(ctx, "kubectl get node -o json") - hosts = [item["metadata"]["name"] for item in getNode["items"]] - optionalHosts = {'slash'} - return { - "name": - "k8s_expected_nodes", - "rules": [{ - "alert": "kube_node_log_size_report_" + h, - "expr": 'absent(kubelet_container_log_filesystem_used_bytes{instance="%s"})' % h, - "for": "1h", - "annotations": { - "summary": f"no recent k8s log size report from host {h}" - }, - } for h in hosts if h not in optionalHosts], - }