Explore the statistics: a query log to drill down by client, name, view and more; storage on the dashboard

jarld keeps every answer for a while (stats.retention.queries, default 7 days) in the statistics database: time, client, view, name, type, result and what blocked it. Every client, name, view, record type, result and blocklist on the Statistics page leads to an explore page for it: answers over time, the top values of every other dimension (each a link that narrows further) and the latest answers. GET /stats/explore.

The dashboard shows the size of Jarl's databases (and the query log's rows) and, per disk Jarl uses, how full it is and the databases' share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/web/static/assets/pages/dashboard.js

@@ -18,6 +18,7 @@ export async function render(el) {
     st.backup ? backupCard(st.backup) : null,
     exposure ? exposureCard(exposure) : null,
     st.certificates ? certCard(st.certificates) : null,
+    st.storage ? storageCard(st.storage) : null,
     st.encrypted_dns ? h("a", { class: "card link", href: "#/encrypted-dns" }, h("div", { class: "label" }, "Encrypted DNS"),
       h("div", { class: "value" }, st.encrypted_dns.ok ? badge("working", "ok") : badge("not working", "err")),
       h("div", { class: "sub" }, "checked ", time(st.encrypted_dns.checked))) : null,
@@ -100,6 +101,7 @@ export async function render(el) {
     h("h2", null, "Configuration checks"),
     real.length ? problemsList(real) : h("div", { class: "msg ok" }, "No problems found in the current configuration."),
     problems.length > real.length ? h("details", null, h("summary", { class: "muted" }, plural(problems.length - real.length, "note")), problemsList(problems.filter((p) => p.severity === "info"))) : null));
+  if (st.storage) parts.push(storagePanel(st.storage));
   if (s.paths) {
     parts.push(h("div", { class: "panel" }, h("h2", null, "Files"), h("dl", { class: "kv" },
       h("dt", null, "named's main config"), h("dd", { class: "mono" }, s.main_config || "–"),
@@ -140,3 +142,44 @@ function certCard(c) {
 function card(label, value, sub) {
   return h("div", { class: "card" }, h("div", { class: "label" }, label), h("div", { class: "value" }, value), sub ? h("div", { class: "sub" }, sub) : null);
 }
+
+// storageCard: the databases' size, and the fullest disk Jarl uses.
+function storageCard(sg) {
+  const fullest = sg.disks.reduce((a, d) => (!a || used(d) > used(a) ? d : a), null);
+  const value = h("span", null, bytes(sg.total));
+  const sub = fullest ? [h("span", { class: used(fullest) > 0.9 ? "warn-text" : "" }, Math.round(used(fullest) * 100) + "% full"), " · " + fullest.path] : "databases";
+  return h("a", { class: "card link", href: "#storage" , onclick: (e) => { e.preventDefault(); document.getElementById("storage")?.scrollIntoView({ behavior: "smooth" }); } },
+    h("div", { class: "label" }, "Databases"), h("div", { class: "value" }, value), h("div", { class: "sub" }, sub));
+}
+
+// storagePanel shows each database's size and, per disk Jarl uses, a bar:
+// the databases, everything else, and what is free.
+function storagePanel(sg) {
+  const dbRows = sg.databases.map((d) => [h("dt", null, d.label),
+    h("dd", null, h("strong", null, bytes(d.bytes)), " ", h("span", { class: "mono small muted" }, d.path),
+      d.query_log_rows != null ? h("div", { class: "small muted" }, "the query log holds " + d.query_log_rows.toLocaleString() + " answers (stats.retention.queries)") : null)]);
+  return h("div", { class: "panel", id: "storage" },
+    h("h2", null, "Storage"),
+    h("dl", { class: "kv" }, dbRows, h("dt", null, "Together"), h("dd", null, h("strong", null, bytes(sg.total)))),
+    sg.disks.map((d) => {
+      const other = Math.max(0, d.total - d.free - d.databases);
+      const pctOf = (n) => (d.total ? (n / d.total) * 100 : 0) + "%";
+      const u = used(d);
+      return h("div", { class: "disk" },
+        h("div", { class: "disk-head" }, h("span", { class: "mono" }, d.path), h("span", { class: "muted small" }, " · " + d.holds.join(", ")),
+          h("span", { class: "grow" }), h("span", { class: u > 0.9 ? "warn-text" : "muted" }, bytes(d.free) + " free of " + bytes(d.total) + " (" + Math.round(u * 100) + "% used)")),
+        h("div", { class: "disk-bar", role: "img", "aria-label": Math.round(u * 100) + "% used, Jarl's databases " + bytes(d.databases) },
+          d.databases ? h("div", { class: "seg db", title: "Jarl's databases: " + bytes(d.databases), style: { width: pctOf(d.databases) } }) : null,
+          h("div", { class: "seg other" + (u > 0.9 ? " full" : ""), title: "Everything else: " + bytes(other), style: { width: pctOf(other) } })),
+        d.databases ? h("div", { class: "small muted" }, h("span", { class: "swatch db" }), " Jarl's databases " + bytes(d.databases), "  ", h("span", { class: "swatch other" }), " everything else") : null);
+    }));
+}
+
+function used(d) { return d.total ? (d.total - d.free) / d.total : 0; }
+
+function bytes(n) {
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  let i = 0;
+  while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; }
+  return (n >= 100 || i === 0 ? Math.round(n) : n.toFixed(1)) + " " + units[i];
+}

internal/web/static/assets/pages/explore.js

@@ -0,0 +1,156 @@
+// Exploring the query log (docs/stats.md#exploring): the answers of the
+// last days, filtered by any mix of client, name, view, record type, result
+// and blocking. Every value is a link that adds it to the filter, so a
+// client leads to its names, a name to the clients asking for it, and so on.
+import { h, mount, fmtTime, errorBox, badge, backLink } from "../ui.js";
+import { get } from "../api.js";
+import { lineChart, chartCard, legend, timeTable, barList, compact } from "../charts.js";
+import { viewsOff } from "./stats.js";
+
+const dims = [
+  ["client", "Clients"], ["name", "Names"], ["view", "Views"],
+  ["type", "Record types"], ["rcode", "Results"], ["blocked", "Blocked by"],
+];
+// What a filter is called, in the chips and the title.
+const filterWords = { client: "client", name: "name", domain: "domain", view: "view", type: "type", rcode: "result", blocked: "blocked" };
+const keys = Object.keys(filterWords);
+const ranges = [["1h", 3600], ["6h", 6 * 3600], ["24h", 86400], ["3d", 3 * 86400], ["7d", 7 * 86400], ["30d", 30 * 86400]];
+
+// exploreHref is the explore page for these filters (and range).
+export function exploreHref(filters, since) {
+  const q = new URLSearchParams();
+  for (const k of keys) if (filters[k]) q.set(k, filters[k]);
+  if (since && since !== "24h") q.set("since", since);
+  return "#/stats/explore" + (q.toString() ? "?" + q : "");
+}
+
+const blockedLabel = (k) => (k === "rules" ? "your rules" : k === "yes" ? "anything" : k === "no" ? "nothing" : k);
+
+export async function render(el, params, query) {
+  const f = {};
+  for (const k of keys) if (query.get(k)) f[k] = query.get(k);
+  const since = query.get("since") || "24h";
+  const with_ = (more) => exploreHref({ ...f, ...more }, since);
+  const without = (k) => { const g = { ...f }; delete g[k]; return exploreHref(g, since); };
+
+  // The Statistics page's view switches apply here too.
+  const off = viewsOff();
+  let views = "";
+  if (off.size) {
+    const all = (await get("/views").catch(() => [])).map((v) => v.name);
+    views = all.filter((v) => !off.has(v)).join(",");
+  }
+  let ex;
+  try {
+    ex = await get("/stats/explore", { ...f, since, views: views || undefined });
+  } catch (e) {
+    mount(el, head(f, without, since, null), e.code === "query_log_off"
+      ? h("div", { class: "msg info" }, e.message) : errorBox(e));
+    return;
+  }
+
+  const parts = [head(f, without, since, ex)];
+  const maxSecs = retentionSecs(ex.retention);
+  parts.push(h("div", { class: "filter-row", role: "group", "aria-label": "Time range" },
+    h("span", { class: "muted" }, "Range"),
+    h("div", { class: "segmented" }, ranges.filter(([, s]) => s <= maxSecs).map(([k]) =>
+      h("a", { href: exploreHref(f, k), class: k === since ? "active" : "" }, k))),
+    h("span", { class: "muted small" }, " · the query log keeps " + ex.retention + (ex.oldest ? ", since " + fmtTime(ex.oldest) : "")),
+    off.size ? h("span", { class: "muted small" }, " · views switched off on the Statistics page are left out") : null));
+
+  const nx = (ex.by.rcode || []).find((e) => e.key === "NXDOMAIN");
+  parts.push(h("div", { class: "cards" },
+    tile("Answers", compact(ex.total), "from " + fmtTime(ex.from)),
+    tile("Blocked", compact(ex.blocked), ex.total ? pct(ex.blocked / ex.total) + " of them" : ""),
+    nx && !f.rcode ? tile("NXDOMAIN", compact(nx.count), pct(nx.count / Math.max(1, ex.total)) + " of them") : null,
+    !f.client && ex.by.client ? tile("Clients", (ex.by.client.length >= 15 ? "15+" : String(ex.by.client.length)), "asking") : null));
+
+  const times = ex.series.map((p) => p.t);
+  const series = [{ label: "Answers", values: ex.series.map((p) => p.count), slot: 1 }];
+  if (ex.blocked) series.push({ label: "Blocked", values: ex.series.map((p) => p.blocked), slot: 2 });
+  parts.push(chartCard(ex.blocked ? "Answers and blocked" : "Answers", "per " + stepWord(ex.step),
+    ex.total ? lineChart({ series, times, step: ex.step, area: true }) : h("div", { class: "empty" }, "No answers match."),
+    ex.total ? timeTable(series, times, ex.step) : null, { wide: true, legend: series.length > 1 ? legend(series) : null }));
+
+  // A breakdown for each dimension the filter leaves open: click a value
+  // to add it.
+  const lists = dims.filter(([d]) => ex.by[d] && !(d === "blocked" && !ex.blocked)).map(([d, title]) => {
+    const href = (k) => with_(d === "blocked" ? { blocked: k } : { [d]: k });
+    const label = d === "blocked" ? blockedLabel : d === "view" ? (k) => (k === "_default" ? "(no views)" : k) : (k) => k;
+    return h("div", { class: "card chart-card" }, h("h3", null, title),
+      barList(ex.by[d], { total: d === "blocked" ? ex.blocked : ex.total, href, label, empty: d === "blocked" ? "Nothing blocked." : "None." }));
+  });
+  if (lists.length) parts.push(h("div", { class: "chart-grid" }, lists));
+
+  parts.push(recentPanel(ex, with_));
+  mount(el, parts);
+}
+
+// head is the title and the filter chips, each with a ✕ to take it out.
+function head(f, without, since, ex) {
+  const chips = keys.filter((k) => f[k]).map((k) => h("span", { class: "chip filter-chip" },
+    h("span", { class: "muted" }, filterWords[k] + " "),
+    h("strong", { class: "mono" }, k === "blocked" ? blockedLabel(f[k]) : f[k]),
+    k === "client" && ex && ex.client_name ? h("span", { class: "muted" }, " " + ex.client_name) : null,
+    " ", h("a", { href: without(k), title: "Take it out", class: "chip-x", "aria-label": "Remove the " + filterWords[k] + " filter" }, "✕")));
+  const extra = [];
+  if (f.name && !f.domain) extra.push(h("a", { href: exploreHref({ ...f, name: undefined, domain: f.name }, since) }, "this name and the names below it"));
+  if ((f.name || f.domain) && (f.blocked === "yes" || (ex && ex.blocked))) extra.push(h("a", { href: "#/blocking?check=" + encodeURIComponent(f.name || f.domain) }, "why is it blocked?"));
+  if (f.name || f.domain) extra.push(h("a", { href: "#/search?q=" + encodeURIComponent(f.name || f.domain) }, "find it in your zones"));
+  return h("div", { class: "page-head" },
+    h("div", { class: "grow" },
+      backLink("Statistics", "#/stats"),
+      h("h1", null, title(f, ex)),
+      h("div", { class: "muted" }, "Every answer in the query log that matches. Click a client, name, view, type or result to narrow it down; ✕ takes a filter out."),
+      chips.length ? h("div", { class: "chips filter-chips" }, chips,
+        h("a", { href: exploreHref({}, since), class: "small" }, "Clear all")) : null,
+      extra.length ? h("div", { class: "small" }, extra.map((a, i) => [i ? " · " : "", a])) : null));
+}
+
+function title(f, ex) {
+  const n = Object.keys(f).length;
+  if (!n) return "Explore the answers";
+  const who = f.client ? (ex && ex.client_name ? ex.client_name : f.client) : "";
+  if (f.client && n === 1) return "Client " + who;
+  const what = [f.rcode, f.type, f.blocked ? "blocked" + (f.blocked !== "yes" ? " by " + blockedLabel(f.blocked) : "") : "",
+    f.name || (f.domain ? f.domain + " and below" : "")].filter(Boolean).join(" ");
+  const parts = [what || "Answers"];
+  if (f.client) parts.push("for " + who);
+  if (f.view) parts.push("in view " + f.view);
+  return parts.join(" ");
+}
+
+// recentPanel lists the latest answers, every cell a link.
+function recentPanel(ex, with_) {
+  if (!ex.recent.length) return null;
+  const a = (text, more, cls) => h("a", { href: with_(more), class: cls || "" }, text);
+  return h("div", { class: "panel" }, h("h2", null, "Latest answers"),
+    h("div", { class: "table-wrap" }, h("table", { class: "recent-queries" },
+      h("thead", null, h("tr", null, ["Time", "Client", "View", "Name", "Type", "Result"].map((t) => h("th", null, t)))),
+      h("tbody", null, ex.recent.map((r) => h("tr", null,
+        h("td", { class: "small muted nowrap" }, fmtTime(new Date(r.ts * 1000).toISOString())),
+        h("td", { class: "nowrap" }, a(r.client_name || r.client, { client: r.client }, "mono"), r.client_name ? h("span", { class: "muted small" }, " " + r.client) : null),
+        h("td", null, a(r.view === "_default" ? "–" : r.view, { view: r.view })),
+        h("td", { class: "oneline-cell" }, a(r.name, { name: r.name }, "mono")),
+        h("td", null, a(r.type, { type: r.type })),
+        h("td", null, r.blocked ? a(badge("blocked: " + blockedLabel(r.blocked), "warn"), { blocked: r.blocked }) : a(r.rcode, { rcode: r.rcode }, r.rcode === "NOERROR" ? "" : "warn-text"))))))),
+    ex.recent.length >= 100 ? h("div", { class: "muted small" }, "The latest 100: narrow the filter to see others.") : null);
+}
+
+function tile(label, value, sub) {
+  return h("div", { class: "card" }, h("div", { class: "label" }, label), h("div", { class: "value" }, value), h("div", { class: "sub" }, sub));
+}
+
+function pct(r) { return (r * 100).toFixed(r < 0.1 ? 1 : 0) + "%"; }
+
+function stepWord(s) {
+  if (s % 86400 === 0) return s === 86400 ? "day" : s / 86400 + " days";
+  if (s % 3600 === 0) return s === 3600 ? "hour" : s / 3600 + " hours";
+  return s === 60 ? "minute" : s / 60 + " minutes";
+}
+
+function retentionSecs(r) {
+  const m = /^(\d+)([dhm])$/.exec(r || "");
+  if (!m) return 7 * 86400;
+  return Number(m[1]) * { d: 86400, h: 3600, m: 60 }[m[2]];
+}

internal/web/static/assets/pages/stats.js

@@ -2,6 +2,7 @@ import { h, mount, fmtTime, relTime, errorBox } from "../ui.js";
 import { get } from "../api.js";
 import { rerender } from "../app.js";
 import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js";
+import { exploreHref } from "./explore.js";
 
 const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"]];
 const rangeSecs = { "1h": 3600, "6h": 6 * 3600, "24h": 86400, "7d": 7 * 86400, "30d": 30 * 86400, "90d": 90 * 86400 };
@@ -21,7 +22,7 @@ let generation = 0;
 // The views shown are remembered per browser: the ones switched off, so a
 // new view shows up switched on.
 const offKey = "jarl.stats.views.off";
-function viewsOff() {
+export function viewsOff() {
   try { return new Set(JSON.parse(localStorage.getItem(offKey) || "[]")); } catch { return new Set(); }
 }
 function saveViewsOff(set) {
@@ -114,17 +115,20 @@ export async function render(el, params, query) {
   parts.push(chartCard("Busiest times", "average queries per hour by weekday and hour of day (local time), last " + ranges.find((r) => r[0] === heatSince)[1],
     heatmap(heat, "queries"), heatmapTable(heat, "queries"), { wide: true }));
 
-  // Who asks for what.
+  // Who asks for what. With the query log, every value leads to the
+  // explore page: a client to what it asked, a name to who asked.
+  const ql = !!(status.retention || {}).queries;
+  const ex = (filters) => (ql ? (k) => exploreHref(filters(k), exploreSince(since)) : null);
   if (status.query_log) {
     const t = data.tops;
     parts.push(h("div", { class: "chart-grid" },
-      listCard("Most queried names", "", barList(t.name, { href: (k) => "#/search?q=" + encodeURIComponent(k) })),
-      listCard("Busiest clients", "", barList(t.client)),
-      listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + "." })),
-      listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + "." })),
-      t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null,
-      t.blocked_client.length ? listCard("Clients with the most blocked queries", "often a smart TV, phone or app phoning home", barList(t.blocked_client)) : null,
-      t.blocked.length ? listCard("Most blocked names", "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: (k) => "#/blocking?check=" + encodeURIComponent(k) })) : null));
+      listCard("Most queried names", ql ? "click one to see who asks for it" : "", barList(t.name, { href: ex((k) => ({ name: k })) || ((k) => "#/search?q=" + encodeURIComponent(k)) })),
+      listCard("Busiest clients", ql ? "click one to see what it asks" : "", barList(t.client, { href: ex((k) => ({ client: k })) })),
+      listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ name: k, rcode: "NXDOMAIN" })) })),
+      listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ client: k, rcode: "REFUSED" })) })),
+      t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail, { href: ex((k) => ({ name: k, rcode: "SERVFAIL" })) })) : null,
+      t.blocked_client.length ? listCard("Clients with the most blocked queries", "often a smart TV, phone or app phoning home", barList(t.blocked_client, { href: ex((k) => ({ client: k, blocked: "yes" })) })) : null,
+      t.blocked.length ? listCard("Most blocked names", ql ? "click a name to see who asks for it, and why it is blocked" : "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: ex((k) => ({ name: k, blocked: "yes" })) || ((k) => "#/blocking?check=" + encodeURIComponent(k)) })) : null));
   } else {
     parts.push(h("div", { class: "msg info" }, "Top names and clients need the response log: set stats.query_log: true in /etc/jarl/jarl.yaml and restart jarl."));
   }
@@ -133,13 +137,13 @@ export async function render(el, params, query) {
   const b = data.breakdowns;
   const rtt = rttOrder.map((k) => b.rtt.find((e) => e.key === k)).filter(Boolean);
   parts.push(h("div", { class: "chart-grid" },
-    listCard("Record types", "", barList(b.qtype, { total: s.queries })),
-    listCard("Result codes", "", barList(b.rcode, { total: s.queries })),
-    b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel })) : null,
+    listCard("Record types", "", barList(b.qtype, { total: s.queries, href: ex((k) => ({ type: k })) })),
+    listCard("Result codes", ql ? "click one to see who gets it, for what" : "", barList(b.rcode, { total: s.queries, href: ex((k) => ({ rcode: k })) })),
+    b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel, href: ex((k) => ({ view: k })) })) : null,
     b.zone.length ? listCard("Zones", "queries per local zone; other = names in no local zone (recursion)", barList(b.zone, { label: (k) => (k === "-" ? "other" : k), href: (k) => (k === "-" ? null : "#/search?q=" + encodeURIComponent(k)) })) : null,
     b.answer.length ? listCard("Answers", "authoritative (your zones) or recursive (from the cache or the internet)", barList(b.answer)) : null,
     rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null,
-    b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: () => "#/blocking" })) : null));
+    b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: ex((k) => ({ blocked: k })) || (() => "#/blocking") })) : null));
 
   parts.push(...serverSection(data.server, complete(data.gauges), rangeLabel));
   el.classList.remove("refreshing");
@@ -302,3 +306,9 @@ function disabledPanel() {
     h("p", { class: "muted" }, "Jarl then makes named publish its counters on a local statistics channel and log responses to /var/log/named, and keeps aggregates " +
       "(never individual queries) in a separate database, by default for 7 days per minute, 90 days per hour and 2 years per day. See docs/stats.md."));
 }
+
+// exploreSince is the Statistics range for the explore page: the query log
+// keeps days, not months.
+function exploreSince(since) {
+  return rangeSecs[since] > 7 * 86400 ? "7d" : since;
+}