groups:
  # docs: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/
  # "Whenever the alert expression results in one or more vector
  # elements at a given point in time, the alert counts as active for
  # these elements' label sets."

  # also https://www.metricfire.com/blog/top-5-prometheus-alertmanager-gotchas/#Missing-metrics
  #
  # any presence of starlette_request_duration_seconds_created{app_name="starlette",method="GET",path="/",status_code="200"} 1.6460176156784086e+09 means someone forgot to set app name

  # - name: webcam
  #   rules:
    # waiting for twinscam revival
      # - alert: twinscam_not_reporting
      #   expr: absent(cam_pipeline_state{job="webcam-record-twinscam"})
      #   for: 2m
      #   labels:
      #     severity: losingData
      #   annotations:
      #     summary: "webcam-record-twinscam is not reporting metrics {{ $labels }}"

      # - alert: cam_garagehall_not_reporting
      #   expr: absent(cam_pipeline_state{job="webcam-record-garagehall"})
      #   for: 2m
      #   labels:
      #     severity: losingData
      #   annotations:
      # #     summary: "webcam-record-garagehall is not reporting metrics {{ $labels }}"

      # - alert: cam_pipeline_stopped
      #   expr: sum without (instance) (cam_pipeline_state{cam_pipeline_state="playing"}) < 1
      #   for: 10m
      #   labels:
      #     severity: losingData
      #   annotations:
      #     summary: "webcam-record gst pipeline is not state=playing {{ $labels }}"

      # - alert: cam_not_advancing
      #   expr: rate(cam_stream_bytes{element="splitmux"}[3m]) < 0.2
      #   for: 10m
      #   labels:
      #     severity: losingData
      #   annotations:
      #     summary: "cam output bytes is advancing too slowly. {{ $labels }}"

      # - alert: webcam_indexer_stalled
      #   expr: rate(webcam_indexer_update_count{job="webcam-indexer"}[5m]) < .01
      #   for: 10m
      #   labels:
      #     severity: webcamUsersAffected
      #   annotations:
      #     summary: "webcam indexer update loop is stalled"

  - name: Outages
    rules:
      - alert: powereagleStalled
        expr: rate(house_power_w[100m]) == 0
        for: 0m
        labels:
          severity: losingData
        annotations:
          summary: "power eagle data stalled"
          description: "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs"

      - alert: powereagleAbsent
        expr: absent_over_time(house_power_w[5m])
        for: 2m
        labels:
          severity: losingData
        annotations:
          summary: "power eagle data missing"
          description: "logs at https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/power-eagle/logs"

      # - alert: wifi_scrape_errors
      #   expr: rate(poll_errors_total{job="wifi"}[2m]) > .1
      #   labels:
      #     severity: houseUsersAffected
      #   annotations:
      #     summary: "errors getting wifi users list"

      # - alert: absent_mitmproxy
      #   expr: absent(process_resident_memory_bytes{job="mitmproxy"})
      #   labels:
      #     severity: houseUsersAffected
      #   annotations:
      #     summary: "mitmproxy metrics not responding. See https://bigasterisk.com/grafana/d/ix3hMAdMk/webfilter?orgId=1&from=now-12h&to=now and https://bigasterisk.com/k/clusters/local/namespaces/default/deployments/mitmproxy (metrics actually come from webfilter.py plugin)"

      - alert: absent_zigbee_dash
        expr: absent(container_last_seen{container="zigbee2mqtt-dash"})

      - alert: net_routes_sync
        expr: rate(starlette_request_duration_seconds_count{app_name="net_routes",path="/routes"}[5m]) < 1/70
        for: 10m
        labels:
          severity: houseUsersAffected
        annotations:
          summary: "net_routes is not getting regular updates"

      
  - name: alerts
    rules:
      - {alert: housePower,   for: 24h, labels: {severity: waste},   expr: "house_power_w > 4000",                                                                annotations: {summary: "house power usage over 3KW {{ $labels }}"}}
      - {alert: disk1,        for: 20m, labels: {severity: warning}, expr: 'disk_free{path=~"/(d[1-9])?"} < 20G',                                                 annotations: {summary: "low disk_free {{ $labels }}"}}
      - {alert: disk2,        for: 20m, labels: {severity: warning}, expr: 'disk_free{path="/stor6/my"} < 100G',                                                  annotations: {summary: "low disk_free {{ $labels }}"}}
      - {alert: disk3,        for: 20m, labels: {severity: warning}, expr: 'round(increase(disk_used{fstype="zfs",path=~"^/stor6.*"}[1w]) / 1M) > 500',           annotations: {summary: "high mb/week on zfs dir {{ $labels }}"}}
      - {alert: oom,          for: 1m,  labels: {severity: warning}, expr: 'predict_linear(mem_free[5m], 5m) / 1M < 100',                                         annotations: {summary: "host about to run OOM {{ $labels }}"}}
      - {alert: high_logging, for: 20m, labels: {severity: waste},   expr: 'sum by (container) (rate(kubelet_container_log_filesystem_used_bytes[30m])) > 30000', annotations: {summary: "high log output rate {{ $labels }}"}}
      - {alert: stale_process, for: 1d, labels: {severity: dataRisk}, expr: 'round((time() - filestat_modification_time/1e9) / 86400) > 14',                       annotations: {summary: "process time is old {{ $labels }}"}}
      - {alert: starlette,    for: 1m,  labels: {severity: fix},     expr: 'starlette_request_duration_seconds_created{app_name="starlette"}',                       annotations: {summary: "set starlette app name {{ $labels }}"}}
      - alert: ssl_certs_expiring_soon
        expr: min((min_over_time(probe_ssl_earliest_cert_expiry[1d])-time())/86400) < 10
        labels:
          severity: warning
        annotations:
          summary: "cert expiring soon. See https://bigasterisk.com/grafana/d/z1YtDa3Gz/certs?orgId=1\nVALUE = {{ $value }}\n  LABELS = {{ $labels }}"