Files @ 06da5db2fafe
Branch filter:

Location: light9/web/metrics/StatsLine.ts - annotation

drewp@bigasterisk.com
rewrite ascoltami to use the graph for more playback data
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
4556eebe5d73
import { css, html, LitElement, TemplateResult } from "lit";
import { customElement, property } from "lit/decorators.js";
export { StatsProcess } from "./StatsProcess";
import parsePrometheusTextFormat from "parse-prometheus-text-format";
import debug from "debug";
import { clamp } from "../floating_color_picker";
const log = debug("home");

interface Value {
  labels: { string: string };
  value?: string;
  count?: number;
  sum?: number;
  buckets?: { [value: string]: string };
}
interface Metric {
  name: string;
  help: string;
  type: "GAUGE" | "SUMMARY" | "COUNTER" | "HISTOGRAM" | "UNTYPED";
  metrics: Value[];
}
type Metrics = Metric[];

function nonBoring(m: Metric) {
  return (
    !m.name.endsWith("_created") && //
    !m.name.startsWith("python_gc_") &&
    m.name != "python_info" &&
    m.name != "process_max_fds" &&
    m.name != "process_virtual_memory_bytes" &&
    m.name != "process_resident_memory_bytes" &&
    m.name != "process_start_time_seconds" &&
    m.name != "process_cpu_seconds_total"
  );
}

@customElement("stats-line")
export class StatsLine extends LitElement {
  @property() name = "?";
  @property() stats: Metrics = [];

  prevCpuNow = 0;
  prevCpuTotal = 0;
  @property() cpu = 0;
  @property() mem = 0;

  updated(changedProperties: any) {
    changedProperties.forEach((oldValue: any, propName: string) => {
      if (propName == "name") {
        const reload = () => {
          fetch("/service/" + this.name + "/metrics").then((resp) => {
            if (resp.ok) {
              resp
                .text()
                .then((msg) => {
                  this.stats = parsePrometheusTextFormat(msg) as Metrics;
                  this.extractProcessStats(this.stats);
                  setTimeout(reload, 1000);
                })
                .catch((err) => {
                  log(`${this.name} failing`, err);
                  setTimeout(reload, 1000);
                });
            } else {
              if (resp.status == 502) {
                setTimeout(reload, 5000);
              }
              // 404: likely not mapped to a responding server
            }
          });
        };
        reload();
      }
    });
  }
  extractProcessStats(stats: Metrics) {
    stats.forEach((row: Metric) => {
      if (row.name == "process_resident_memory_bytes") {
        this.mem = parseFloat(row.metrics[0].value!) / 1024 / 1024;
      }
      if (row.name == "process_cpu_seconds_total") {
        const now = Date.now() / 1000;
        const cpuSecondsTotal = parseFloat(row.metrics[0].value!);
        this.cpu = (cpuSecondsTotal - this.prevCpuTotal) / (now - this.prevCpuNow);
        this.prevCpuTotal = cpuSecondsTotal;
        this.prevCpuNow = now;
      }
    });
  }

  static styles = [
    css`
      :host {
        border: 2px solid #46a79f;
        display: inline-block;
      }
      table {
        border-collapse: collapse;
        background: #000;
        color: #ccc;
        font-family: sans-serif;
      }
      th,
      td {
        outline: 1px solid #000;
      }
      th {
        padding: 2px 4px;
        background: #2f2f2f;
        text-align: left;
      }
      td {
        padding: 0;
        vertical-align: top;
        text-align: center;
      }
      td.val {
        padding: 2px 4px;
        background: #3b5651;
      }
      .recents {
        display: flex;
        align-items: flex-end;
        height: 30px;
      }
      .recents > div {
        width: 3px;
        background: red;
        border-right: 1px solid black;
      }
      .bigInt {
        min-width: 6em;
      }
    `,
  ];

  tdWrap(content: TemplateResult): TemplateResult {
    return html`<td>${content}</td>`;
  }

  recents(d: any, path: string[]): TemplateResult {
    const hi = Math.max.apply(null, d.recents);
    const scl = 30 / hi;

    const bar = (y: number) => {
      let color;
      if (y < d.average) {
        color = "#6a6aff";
      } else {
        color = "#d09e4c";
      }
      return html`<div class="bar" style="height: ${y * scl}px; background: ${color};"></div>`;
    };
    return html`<td>
      <div class="recents">${d.recents.map(bar)}</div>
      <div>avg=${d.average.toPrecision(3)}</div>
    </td>`;
  }

  table(d: Metrics, path: string[]): TemplateResult {
    const byName = new Map<string, Metric>();
    d.forEach((row) => {
      byName.set(row.name, row);
    });
    let cols = d.map((row) => row.name);
    cols.sort();

    if (path.length == 0) {
      ["webServer", "process"].forEach((earlyKey) => {
        let i = cols.indexOf(earlyKey);
        if (i != -1) {
          cols = [earlyKey].concat(cols.slice(0, i), cols.slice(i + 1));
        }
      });
    }

    const th = (col: string): TemplateResult => {
      return html`<th>${col}</th>`;
    };
    const td = (col: string): TemplateResult => {
      const cell = byName.get(col)!;
      return html`${this.drawLevel(cell, path.concat(col))}`;
    };
    return html` <table>
      <tr>
        ${cols.map(th)}
      </tr>
      <tr>
        ${cols.map(td)}
      </tr>
    </table>`;
  }

  drawLevel(d: Metric, path: string[]) {
    return html`[NEW ${JSON.stringify(d)} ${path}]`;
  }

  valueDisplay(m: Metric, v: Value): TemplateResult {
    if (m.type == "GAUGE") {
      return html`${v.value}`;
    } else if (m.type == "COUNTER") {
      return html`${v.value}`;
    } else if (m.type == "HISTOGRAM") {
      return this.histoDisplay(v.buckets!);
    } else if (m.type == "UNTYPED") {
      return html`${v.value}`;
    } else if (m.type == "SUMMARY") {
      if (!v.count) {
        return html`err: summary without count`;
      }
      return html`n=${v.count} percall=${((v.count && v.sum ? v.sum / v.count : 0) * 1000).toPrecision(3)}ms`;
    } else {
      throw m.type;
    }
  }

  private histoDisplay(b: { [value: string]: string }) {
    const lines: TemplateResult[] = [];
    let firstLevel;
    let lastLevel;
    let prev = 0;

    let maxDelta = 0;
    for (let level in b) {
      if (firstLevel === undefined) firstLevel = level;
      lastLevel = level;
      let count = parseFloat(b[level]);
      let delta = count - prev;
      prev = count;
      if (delta > maxDelta) maxDelta = delta;
    }
    prev = 0;
    const maxBarH = 30;
    for (let level in b) {
      let count = parseFloat(b[level]);
      let delta = count - prev;
      prev = count;
      let levelf = parseFloat(level);
      const h = clamp((delta / maxDelta) * maxBarH, 1, maxBarH);
      lines.push(
        html`<div
          title="bucket=${level} count=${count}"
          style="background: yellow; margin-right: 1px; width: 8px; height: ${h}px; display: inline-block"
        ></div>`
      );
    }
    return html`${firstLevel} ${lines} ${lastLevel}`;
  }

  tightLabel(labs: { [key: string]: string }): string {
    const d: { [key: string]: string } = {};
    for (let k in labs) {
      if (k == "app_name") continue;
      if (k == "output") continue;
      if (k == "status_code" && labs[k] == "200") continue;
      d[k] = labs[k];
    }
    const ret = JSON.stringify(d);
    return ret == "{}" ? "" : ret;
  }
  tightMetric(name: string): string {
    return name.replace("starlette", "⭐").replace("_request", "_req").replace("_duration", "_dur").replace("_seconds", "_s");
  }
  render() {
    const now = Date.now() / 1000;

    const displayedStats = this.stats.filter(nonBoring);
    return html`
      <div>
        <table>
          ${displayedStats.map(
            (row, rowNum) => html`
              <tr>
                <th>${this.tightMetric(row.name)}</th>
                <td>
                  <table>
                    ${row.metrics.map(
                      (v) => html`
                        <tr>
                          <td>${this.tightLabel(v.labels)}</td>
                          <td>${this.valueDisplay(row, v)}</td>
                        </tr>
                      `
                    )}
                  </table>
                </td>
                ${rowNum == 0
                  ? html`
                      <td rowspan="${displayedStats.length}">
                        <stats-process id="proc" cpu="${this.cpu}" mem="${this.mem}"></stats-process>
                      </td>
                    `
                  : ""}
              </tr>
            `
          )}
        </table>
      </div>
    `;
  }
}