jarl/internal/web/static/assets/pages/stats.js

import { h, mount, fmtTime, relTime, errorBox } from "../ui.js";
import { get } from "../api.js";
import { rerender } from "../app.js";
import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js";
import { exploreHref } from "./explore.js";

const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"]];
const rangeSecs = { "1h": 3600, "6h": 6 * 3600, "24h": 86400, "7d": 7 * 86400, "30d": 30 * 86400, "90d": 90 * 86400 };

// Failed answers keep the same color whatever the range: color follows
// the result code, never its rank.
const failures = [
  { key: "rcode:NXDOMAIN", label: "NXDOMAIN", slot: 1 },
  { key: "rcode:SERVFAIL", label: "SERVFAIL", slot: 2 },
  { key: "rcode:REFUSED", label: "REFUSED", slot: 3 },
];

const rttOrder = ["<10ms", "10-100ms", "100-500ms", "500-800ms", "800-1600ms", ">1600ms"];

let generation = 0;

// The views shown are remembered per browser: the ones switched off, so a
// new view shows up switched on.
const offKey = "jarl.stats.views.off";
export function viewsOff() {
  try { return new Set(JSON.parse(localStorage.getItem(offKey) || "[]")); } catch { return new Set(); }
}
function saveViewsOff(set) {
  try { localStorage.setItem(offKey, JSON.stringify([...set])); } catch { /* private window: not remembered */ }
}

// Breakdowns that exist per view; the others are named's own counters,
// for the whole server.
const perViewGroups = ["qtype", "rcode", "view", "zone", "blocklist"];

export async function render(el, params, query) {
  const gen = ++generation;
  const want = query && query.get ? query.get("since") : null;
  const since = rangeSecs[want] ? want : "24h";
  const status = await get("/stats/status");
  const head = h("div", { class: "page-head" },
    h("div", { class: "grow" }, h("h1", null, "Statistics"),
      h("div", { class: "muted" }, status.enabled
        ? "From named's counters" + (status.query_log ? " and response log" : "") + " · data since " + (status.first_data ? fmtTime(status.first_data) : "–") +
          " · kept: minutes " + status.retention.minute + ", hours " + status.retention.hour + ", days " + status.retention.day
        : "Query statistics are off")));
  if (!status.enabled) {
    mount(el, head, disabledPanel());
    return;
  }
  const heatSince = rangeSecs[since] < 7 * 86400 ? "7d" : since;
  // Which views: all of them unless some are switched off.
  const allViews = status.query_log ? (await get("/views").catch(() => [])).map((v) => v.name) : [];
  const off = viewsOff();
  const shown = allViews.filter((v) => !off.has(v));
  const filtered = allViews.length > 0 && shown.length < allViews.length;
  const vf = filtered ? { views: shown.join(",") } : {};
  if (filtered && !shown.length) {
    mount(el, head, filterRow(since, allViews, off), h("div", { class: "empty" }, "No views chosen: switch one on above."));
    return;
  }
  let data;
  try {
    data = await loadAll(since, heatSince, status.query_log, vf, filtered);
  } catch (e) {
    mount(el, head, filterRow(since, allViews, off), errorBox(e));
    return;
  }
  if (gen !== generation) return; // the user moved on
  const { summary: s, heat } = data;
  // The newest bucket is still filling up; drawing it would look like a
  // sudden drop in traffic.
  const timeline = complete(data.timeline), errors = complete(data.errors);
  const step = stepSeconds[timeline.step];
  const parts = [head, filterRow(since, allViews, off)];
  for (const w of status.warnings || []) parts.push(h("div", { class: "msg warn" }, w));
  if (filtered) {
    parts.push(h("div", { class: "msg info small" }, "Showing view" + (shown.length > 1 ? "s " : " ") + shown.join(", ") + ": answers from named's response log" +
      (s.views_since ? ", per view since " + fmtTime(s.views_since) : "") + ". named's own figures (answer sources, upstream times, cache, the server) are for the whole server."));
  }

  // Headline numbers.
  const rangeLabel = "in the last " + ranges.find((r) => r[0] === since)[1];
  parts.push(h("div", { class: "cards" },
    tile(filtered ? "Answers" : "Queries", compact(s.queries), rangeLabel),
    tile("Per second", s.avg_qps < 10 ? s.avg_qps.toFixed(2) : Math.round(s.avg_qps).toLocaleString(), "on average"),
    s.peak ? tile("Busiest " + s.peak.per, compact(s.peak.queries), fmtTime(s.peak.time)) : null,
    s.nxdomain_ratio != null ? tile("NXDOMAIN", pct(s.nxdomain_ratio), "of queries named no existing name") : null,
    s.cache_hit_ratio != null ? tile("Cache hits", pct(s.cache_hit_ratio), "of recursive queries answered from cache") : null,
    s.blocked_ratio != null ? tile("Blocked", pct(s.blocked_ratio), compact(s.blocked) + " queries, by your rules and blocklists") : null));

  // Volume over time: one series, so no legend box; the title names it.
  // With blocking, a second line shows how many of them were blocked.
  const q = timeline.series.find((x) => x.key === "queries") || { values: timeline.times.map(() => 0) };
  const qSeries = [{ label: filtered ? "Answers" : "Queries", values: q.values, slot: 1 }];
  const bl = timeline.series.find((x) => x.key === "blocked");
  if (bl && bl.values.some((v) => v > 0)) qSeries.push({ label: "Blocked", values: bl.values, slot: 2 });
  // Failed answers: three fixed series, legend plus end labels.
  const fSeries = failures.map((f) => ({ ...f, values: (errors.series.find((x) => x.key === f.key) || { values: errors.times.map(() => 0) }).values }));
  const anyFailures = fSeries.some((f) => f.values.some((v) => v > 0));
  // The two charts are one above the other: the same room for labels on
  // both sides keeps their time axes in line.
  const gutter = { left: 52, right: Math.max(endLabelWidth(qSeries), anyFailures ? endLabelWidth(fSeries) : 0) };
  const qWord = filtered ? "Answers" : "Queries";
  parts.push(chartCard(qSeries.length > 1 ? qWord + " and blocked queries" : qWord, "per " + stepName(timeline.step) + ", " + rangeLabel,
    lineChart({ series: qSeries, times: timeline.times, step, area: true, gutter }), timeTable(qSeries, timeline.times, step),
    { wide: true, legend: qSeries.length > 1 ? legend(qSeries) : null }));

  parts.push(chartCard("Failed answers", "NXDOMAIN: the name doesn't exist · SERVFAIL: resolution failed · REFUSED: not allowed",
    anyFailures ? lineChart({ series: fSeries, times: errors.times, step: stepSeconds[errors.step], gutter }) : h("div", { class: "empty" }, "No failed answers " + rangeLabel + "."),
    anyFailures ? timeTable(fSeries, errors.times, stepSeconds[errors.step]) : null,
    { wide: true, legend: anyFailures ? legend(fSeries) : null }));

  // When is the server busy?
  parts.push(chartCard("Busiest times", "average queries per hour by weekday and hour of day (local time), last " + ranges.find((r) => r[0] === heatSince)[1],
    heatmap(heat, "queries"), heatmapTable(heat, "queries"), { wide: true }));

  // Who asks for what. With the query log, every value leads to the
  // explore page: a client to what it asked, a name to who asked.
  const ql = !!(status.retention || {}).queries;
  const ex = (filters) => (ql ? (k) => exploreHref(filters(k), exploreSince(since)) : null);
  if (status.query_log) {
    const t = data.tops;
    parts.push(h("div", { class: "chart-grid" },
      listCard("Most queried names", ql ? "click one to see who asks for it" : "", barList(t.name, { href: ex((k) => ({ name: k })) || ((k) => "#/search?q=" + encodeURIComponent(k)) })),
      listCard("Busiest clients", ql ? "click one to see what it asks" : "", barList(t.client, { href: ex((k) => ({ client: k })) })),
      listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ name: k, rcode: "NXDOMAIN" })) })),
      listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ client: k, rcode: "REFUSED" })) })),
      t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail, { href: ex((k) => ({ name: k, rcode: "SERVFAIL" })) })) : null,
      t.blocked_client.length ? listCard("Clients with the most blocked queries", "often a smart TV, phone or app phoning home", barList(t.blocked_client, { href: ex((k) => ({ client: k, blocked: "yes" })) })) : null,
      t.blocked.length ? listCard("Most blocked names", ql ? "click a name to see who asks for it, and why it is blocked" : "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: ex((k) => ({ name: k, blocked: "yes" })) || ((k) => "#/blocking?check=" + encodeURIComponent(k)) })) : null));
  } else {
    parts.push(h("div", { class: "msg info" }, "Top names and clients need the response log: set stats.query_log: true in /etc/jarl/jarl.yaml and restart jarl."));
  }

  // Breakdowns.
  const b = data.breakdowns;
  const rtt = rttOrder.map((k) => b.rtt.find((e) => e.key === k)).filter(Boolean);
  parts.push(h("div", { class: "chart-grid" },
    listCard("Record types", "", barList(b.qtype, { total: s.queries, href: ex((k) => ({ type: k })) })),
    listCard("Result codes", ql ? "click one to see who gets it, for what" : "", barList(b.rcode, { total: s.queries, href: ex((k) => ({ rcode: k })) })),
    b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel, href: ex((k) => ({ view: k })) })) : null,
    b.zone.length ? listCard("Zones", "queries per local zone; other = names in no local zone (recursion)", barList(b.zone, { label: (k) => (k === "-" ? "other" : k), href: (k) => (k === "-" ? null : "#/search?q=" + encodeURIComponent(k)) })) : null,
    b.answer.length ? listCard("Answers", "authoritative (your zones) or recursive (from the cache or the internet)", barList(b.answer)) : null,
    rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null,
    b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: ex((k) => ({ blocked: k })) || (() => "#/blocking") })) : null));

  parts.push(...serverSection(data.server, complete(data.gauges), rangeLabel));
  el.classList.remove("refreshing");
  mount(el, parts);
  scheduleRefresh(el, params, query, gen);
}

async function loadAll(since, heatSince, queryLog, vf, filtered) {
  const p = {
    summary: get("/stats/summary", { since, top: 0, ...vf }),
    timeline: get("/stats/timeline", { since, key: "queries,blocked", ...vf }),
    errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(","), ...vf }),
    heat: get("/stats/heatmap", { since: heatSince, ...vf }),
    server: get("/stats/server").catch(() => null),
    gauges: get("/stats/timeline", { since, key: gaugeKeys.join(",") }),
  };
  const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt", "blocklist"];
  // With some views only, named's own counters aren't asked for.
  groups.forEach((g) => { p["b_" + g] = filtered && !perViewGroups.includes(g) ? Promise.resolve([]) : get("/stats/breakdown", { group: g, since, ...vf }); });
  if (queryLog) {
    for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked", "blocked_client"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15, ...vf });
  }
  const keys = Object.keys(p);
  const vals = await Promise.all(Object.values(p));
  const out = { breakdowns: {}, tops: {} };
  keys.forEach((k, i) => {
    if (k.startsWith("b_")) out.breakdowns[k.slice(2)] = vals[i];
    else if (k.startsWith("t_")) out.tops[k.slice(2)] = vals[i];
    else out[k] = vals[i];
  });
  return out;
}

// scheduleRefresh reloads the page's data every minute while it is shown,
// keeping the current render (dimmed) until the new one is ready.
function scheduleRefresh(el, params, query, gen) {
  setTimeout(() => {
    if (gen !== generation || !location.hash.startsWith("#/stats") || !el.isConnected) return;
    el.classList.add("refreshing");
    render(el, params, query).catch(() => el.classList.remove("refreshing"));
  }, 60_000);
}

// complete drops the last bucket of a timeline if it hasn't ended yet.
function complete(tl) {
  const step = stepSeconds[tl.step];
  const n = tl.times.length;
  if (n > 1 && tl.times[n - 1] + step > Date.now() / 1000) {
    return { ...tl, times: tl.times.slice(0, -1), series: tl.series.map((x) => ({ ...x, values: x.values.slice(0, -1) })) };
  }
  return tl;
}

function filterRow(since, views, off) {
  const range = h("div", { class: "filter-row", role: "group", "aria-label": "Time range" },
    h("span", { class: "muted" }, "Range"),
    h("div", { class: "segmented" }, ranges.map(([k, label]) =>
      h("a", { href: "#/stats?since=" + k, class: k === since ? "active" : "", "aria-current": k === since ? "true" : null, title: "last " + label }, k))));
  if (!views || views.length < 2) return range;
  // A toggle per view (together: linked views, say), and all or none.
  const set = (next) => { saveViewsOff(next); rerender(); };
  const toggles = views.map((v) => {
    const on = !off.has(v);
    return h("button", { type: "button", class: "chip toggle-chip" + (on ? " on" : ""), "aria-pressed": on ? "true" : "false",
      title: on ? "Leave view " + v + " out" : "Include view " + v,
      onclick: () => { const n = new Set(off); if (on) n.add(v); else n.delete(v); set(n); } }, v);
  });
  return h("div", { class: "filter-rows" }, range,
    h("div", { class: "filter-row", role: "group", "aria-label": "Views" },
      h("span", { class: "muted" }, "Views"),
      h("div", { class: "chips" }, toggles),
      h("button", { type: "button", class: "small", disabled: off.size === 0, onclick: () => set(new Set()) }, "All"),
      h("button", { type: "button", class: "small", disabled: off.size === views.length, onclick: () => set(new Set(views)) }, "None")));
}

const gaugeKeys = ["g:named:rss", "g:named:mem", "g:cache:mem", "g:named:cpu"];

// serverSection shows named's process, cache, clients, the host and the
// disks now, and named's memory and CPU over time.
function serverSection(sv, gauges, rangeLabel) {
  const srv = (sv && sv.server) || {};
  const named = (sv && sv.named) || {};
  const host = srv.host || {};
  const out = [h("h2", { class: "section" }, "Server")];
  const clients = (v, what) => {
    // "now/soft/hard" or "now/limit".
    const [now, ...limits] = (v || "").split("/");
    return v ? [now, "of " + limits[limits.length - 1] + " allowed" + what] : null;
  };
  const rc = clients(named.recursive_clients, srv.recursion_highwater ? " · most at once: " + srv.recursion_highwater : "");
  const tc = clients(named.tcp_clients, srv.tcp_highwater ? " · most at once: " + srv.tcp_highwater : "");
  out.push(h("div", { class: "cards" },
    tile("named memory", srv.rss ? bytes(srv.rss) : "–", srv.rss ? "resident · named counts " + bytes(srv.mem_in_use) + " · " + srv.threads + " threads" : "named's process isn't visible to jarld"),
    tile("named CPU", srv.pid ? srv.cpu_percent.toFixed(1) + "%" : "–", "of one CPU, last minute" + (named.cpus ? " · " + named.cpus + " CPUs" : "")),
    tile("Cache", compact(srv.cache_rrsets || 0) + " RRsets", bytes(srv.cache_mem || 0) + " of memory, all views"),
    rc ? tile("Recursive clients", rc[0], rc[1]) : null,
    tc ? tile("TCP clients", tc[0], tc[1]) : null,
    tile("Up since", srv.boot_time ? relTime(srv.boot_time).replace(" ago", "") : "–", srv.config_time ? "named started " + fmtTime(srv.boot_time) + " · reconfigured " + relTime(srv.config_time) : ""),
    host.mem_total ? tile("Host memory", pct((host.mem_total - host.mem_available) / host.mem_total), "of " + bytes(host.mem_total) + " in use · load " + (host.load || []).map((x) => x.toFixed(2)).join(" ")) : null,
    ...((sv && sv.disks) || []).map((d) => tile("Disk space", bytes(d.free) + " free", h("span", { title: d.path }, "of " + bytes(d.total) + ", for " + d.holds.join(", "))))));
  if (sv && sv.named_error) out.push(h("div", { class: "msg info" }, "Client counts need rndc: " + sv.named_error));
  // Gauges over time: averages, -1 where there was no sample.
  const series = (key) => ((gauges.series || []).find((x) => x.key === key) || { values: gauges.times.map(() => -1) }).values.map((v) => (v < 0 ? null : v));
  const gStep = stepSeconds[gauges.step];
  const memSeries = [
    { label: "Resident", values: series("g:named:rss"), slot: 1 },
    { label: "named's count", values: series("g:named:mem"), slot: 2 },
    { label: "Cache", values: series("g:cache:mem"), slot: 3 },
  ];
  const cpuSeries = [{ label: "CPU", values: series("g:named:cpu").map((v) => (v == null ? null : v / 10)), slot: 1 }];
  const any = (ss) => ss.some((x) => x.values.some((v) => v != null));
  const gutter = { left: 64, right: endLabelWidth(memSeries) };
  out.push(chartCard("named memory", "average per " + stepName(gauges.step) + ", " + rangeLabel + " · resident is what the system sees",
    any(memSeries) ? lineChart({ series: memSeries, times: gauges.times, step: gStep, gutter, format: bytes }) : h("div", { class: "empty" }, "No samples yet: they are taken once a minute."),
    any(memSeries) ? timeTable(memSeries, gauges.times, gStep, bytes) : null, { wide: true, legend: any(memSeries) ? legend(memSeries) : null }));
  out.push(chartCard("named CPU", "percent of one CPU, average per " + stepName(gauges.step),
    any(cpuSeries) ? lineChart({ series: cpuSeries, times: gauges.times, step: gStep, gutter, format: (v) => (Math.round(v * 10) / 10) + "%" }) : h("div", { class: "empty" }, "No samples yet."),
    any(cpuSeries) ? timeTable(cpuSeries, gauges.times, gStep, (v) => v.toFixed(1) + "%") : null, { wide: true }));
  return out;
}

// bytes formats a size in binary units.
function bytes(n) {
  const units = ["B", "KiB", "MiB", "GiB", "TiB"];
  let i = 0;
  while (n >= 1024 && i < units.length - 1) {
    n /= 1024;
    i++;
  }
  return (i === 0 || n >= 100 ? Math.round(n) : n.toFixed(1).replace(/\.0$/, "")) + " " + units[i];
}

function tile(label, value, sub) {
  return h("div", { class: "card stat" }, h("div", { class: "label" }, label), h("div", { class: "value big" }, value), h("div", { class: "sub" }, sub));
}

function listCard(title, subtitle, body) {
  return h("section", { class: "panel" }, h("h2", null, title), subtitle ? h("div", { class: "muted small list-sub" }, subtitle) : null, body);
}

function pct(r) {
  const p = r * 100;
  return (p < 10 ? p.toFixed(1) : Math.round(p)) + "%";
}

function stepName(step) {
  return { "1m": "minute", "5m": "5 minutes", "15m": "15 minutes", "1h": "hour", "6h": "6 hours", "1d": "day" }[step] || step;
}

function viewLabel(k) {
  return k === "_default" ? "(no view)" : k;
}

function disabledPanel() {
  return h("div", { class: "panel" },
    h("h2", null, "Turn on query statistics"),
    h("p", null, "Jarl can show how many queries named answers over time, the busiest hours, the most queried names and the clients asking most. " +
      "It is off by default. To turn it on, add this to /etc/jarl/jarl.yaml on the server and restart jarl (sudo systemctl restart jarl):"),
    h("pre", { class: "code" }, "stats:\n  enabled: true"),
    h("p", { class: "muted" }, "Jarl then makes named publish its counters on a local statistics channel and log responses to /var/log/named, and keeps aggregates " +
      "(never individual queries) in a separate database, by default for 7 days per minute, 90 days per hour and 2 years per day. See docs/stats.md."));
}

// exploreSince is the Statistics range for the explore page: the query log
// keeps days, not months.
function exploreSince(since) {
  return rangeSecs[since] > 7 * 86400 ? "7d" : since;
}