From 1b102a78f6b4fcb5ebf271c31fd5243e6ae71929 Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 13 Oct 2026 18:24:32 +0100 Subject: [PATCH] Statistics page: the statistics database's size, and a fullness bar on the disk tiles The server section shows the statistics database's size (with its -wal and -shm) and how many answers the query log holds, linking to the dashboard's Storage panel; the disk tiles get the same bar as the dashboard. GET /stats/server has database {path, bytes, query_log_rows}. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- docs/api.md | 2 +- docs/stats.md | 6 +++-- internal/server/h_stats.go | 10 ++++++++ internal/server/h_storage.go | 18 +++++++++----- internal/server/stats_test.go | 22 +++++++++++++++++ internal/web/static/assets/pages/stats.js | 29 ++++++++++++++++++++--- internal/web/static/assets/style.css | 1 + 8 files changed, 78 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c66f1ec..294e040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,8 @@ All notable changes to Jarl are documented here. The format follows the clients asking → everything one of them asks. `GET /stats/explore`. - **Storage** on the dashboard: the size of Jarl's databases (and how many answers the query log holds), and per disk Jarl uses a bar of how full it - is and how much of it the databases take. + is and how much of it the databases take. The Statistics page shows the + statistics database's size too, and its disk tiles get the same bar. ### Changed diff --git a/docs/api.md b/docs/api.md index ffde06b..a34abef 100644 --- a/docs/api.md +++ b/docs/api.md @@ -566,7 +566,7 @@ Needs `stats:read`. When statistics are disabled, every endpoint but | GET | `/stats/top` | `kind` (name, client, nxdomain, servfail, refused, blocked, blocked_client), `since`, `limit` (default 20) | `[{key, count}]`; client, refused and blocked_client entries add `name`, the host name of Jarl's record for the address (the summary's client list too) | | GET | `/stats/heatmap` | `since` (default 28d), `key` (default queries) | `{key, from, avg[7][24], samples[7][24]}`, Monday first, local time, `-1` = no data | | GET | `/stats/explore` | `?client=&name=&domain=&view=&type=&rcode=&blocked=yes\|no\|LIST&since=24h&views=` | The query log for a filter ([stats.md](stats.md#exploring)): `total`, `blocked`, `series` (per `step` seconds: `t`, `count`, `blocked`), `by` (top 15 of each dimension the filter leaves open: client, name, view, type, rcode, blocked; clients with `name`), `recent` (latest 100: ts, client, client_name, view, name, type, rcode, blocked), `oldest`, `retention`, `client_name`. 404 `query_log_off` without a query log | -| GET | `/stats/server` | | `{server: {rss, mem_in_use, cpu_percent, threads, pid, cache_rrsets, cache_mem, sockets_udp, sockets_tcp, recursion_highwater, tcp_highwater, version, boot_time, config_time, host: {mem_total, mem_available, load, cpus, uptime}}, named: {recursive_clients, tcp_clients, zones, cpus, worker_threads}, disks: [{path, holds, total, free}]}`; gauges over time: `/stats/timeline?key=g:named:rss` (averages, -1 without a sample) | +| GET | `/stats/server` | | `{server: {rss, mem_in_use, cpu_percent, threads, pid, cache_rrsets, cache_mem, sockets_udp, sockets_tcp, recursion_highwater, tcp_highwater, version, boot_time, config_time, host: {mem_total, mem_available, load, cpus, uptime}}, named: {recursive_clients, tcp_clients, zones, cpus, worker_threads}, disks: [{path, holds, total, free}], database: {path, bytes, query_log_rows}}`; gauges over time: `/stats/timeline?key=g:named:rss` (averages, -1 without a sample) | | GET | `/stats/breakdown` | `group` (qtype, rcode, view, zone, answer, transport, rtt, dnssec, cache, ratelimit, blocklist), `since` | `[{key, count}]` | ```sh diff --git a/docs/stats.md b/docs/stats.md index cec2a3a..8a0e6d8 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -221,8 +221,10 @@ view switches apply here too. `anonymize_clients` and `exclude_clients` apply to the query log as to everything else. Its size: each answer takes about 150 to 200 bytes with its indexes, so 50,000 answers a day for 7 days is about 60 MB, 200,000 a -day about 250 MB. The dashboard's Storage panel shows the databases' size -and how many answers the query log holds. +day about 250 MB. The Statistics page's server section shows the +statistics database's size and how many answers the query log holds, next +to the disks (with a bar of how full each is); the dashboard's Storage +panel has all of Jarl's databases. ## Views diff --git a/internal/server/h_stats.go b/internal/server/h_stats.go index 79aabf4..44afc80 100644 --- a/internal/server/h_stats.go +++ b/internal/server/h_stats.go @@ -283,6 +283,16 @@ func (s *Server) statsServer(w http.ResponseWriter, r *http.Request, p *auth.Pri labels = append(labels, "statistics database") } out["disks"] = stats.DiskUsage(labels, dirs) + // The statistics database: what the statistics cost on disk. + if db := s.cfg.Stats.Store().Path(); db != "" && db != ":memory:" { + info := map[string]any{"path": db, "bytes": dbBytes(db)} + if keep := s.cfg.Stats.Options().RetainQueries; keep > 0 && s.cfg.Stats.Options().QueryLog { + if n, err := s.cfg.Stats.Store().QueryLogSize(r.Context()); err == nil { + info["query_log_rows"] = n + } + } + out["database"] = info + } writeJSON(w, http.StatusOK, out) return nil } diff --git a/internal/server/h_storage.go b/internal/server/h_storage.go index 012445a..d8139b1 100644 --- a/internal/server/h_storage.go +++ b/internal/server/h_storage.go @@ -18,12 +18,7 @@ func (s *Server) storage(ctx context.Context) *api.Storage { if path == "" || path == ":memory:" { return nil } - d := api.Database{Label: label, Path: path} - for _, suffix := range []string{"", "-wal", "-shm"} { - if fi, err := os.Stat(path + suffix); err == nil { - d.Bytes += fi.Size() - } - } + d := api.Database{Label: label, Path: path, Bytes: dbBytes(path)} out.Databases = append(out.Databases, d) out.Total += d.Bytes return &out.Databases[len(out.Databases)-1] @@ -70,3 +65,14 @@ func (s *Server) storage(ctx context.Context) *api.Storage { } return out } + +// dbBytes is the size of a SQLite database: its file, -wal and -shm. +func dbBytes(path string) int64 { + var n int64 + for _, suffix := range []string{"", "-wal", "-shm"} { + if fi, err := os.Stat(path + suffix); err == nil { + n += fi.Size() + } + } + return n +} diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go index 8aa6145..cf3b107 100644 --- a/internal/server/stats_test.go +++ b/internal/server/stats_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "log/slog" + "path/filepath" "strings" "testing" "time" @@ -176,3 +177,24 @@ func TestStatsExploreAPI(t *testing.T) { t.Errorf("storage %v", st["storage"]) } } + +// /stats/server reports the statistics database's size and the query log. +func TestStatsServerDatabase(t *testing.T) { + ss, err := stats.OpenStore(filepath.Join(t.TempDir(), "stats.db")) + if err != nil { + t.Fatal(err) + } + defer ss.Close() + b := &stats.Batch{} + b.AddQueries([]stats.QueryRow{{Time: time.Now().Unix(), Client: "10.0.0.7", View: "internal", Name: "www.example.com", Type: "A", RCode: "NOERROR"}}) + if err := ss.Commit(context.Background(), b); err != nil { + t.Fatal(err) + } + opts := stats.Options{QueryLog: true, StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour, RetainQueries: 7 * 24 * time.Hour} + e := newAPIEnv(t, stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))) + sv := e.do("GET", "/stats/server", e.token, nil, 200) + db, ok := sv["database"].(map[string]any) + if !ok || db["bytes"].(float64) <= 0 || db["query_log_rows"].(float64) != 1 { + t.Errorf("database %v", sv["database"]) + } +} diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js index 74aad01..7bf321f 100644 --- a/internal/web/static/assets/pages/stats.js +++ b/internal/web/static/assets/pages/stats.js @@ -145,7 +145,7 @@ export async function render(el, params, query) { rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null, b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: ex((k) => ({ blocked: k })) || (() => "#/blocking") })) : null)); - parts.push(...serverSection(data.server, complete(data.gauges), rangeLabel)); + parts.push(...serverSection(data.server, complete(data.gauges), rangeLabel, status)); el.classList.remove("refreshing"); mount(el, parts); scheduleRefresh(el, params, query, gen); @@ -223,7 +223,7 @@ const gaugeKeys = ["g:named:rss", "g:named:mem", "g:cache:mem", "g:named:cpu"]; // serverSection shows named's process, cache, clients, the host and the // disks now, and named's memory and CPU over time. -function serverSection(sv, gauges, rangeLabel) { +function serverSection(sv, gauges, rangeLabel, status) { const srv = (sv && sv.server) || {}; const named = (sv && sv.named) || {}; const host = srv.host || {}; @@ -243,7 +243,8 @@ function serverSection(sv, gauges, rangeLabel) { tc ? tile("TCP clients", tc[0], tc[1]) : null, tile("Up since", srv.boot_time ? relTime(srv.boot_time).replace(" ago", "") : "–", srv.config_time ? "named started " + fmtTime(srv.boot_time) + " · reconfigured " + relTime(srv.config_time) : ""), host.mem_total ? tile("Host memory", pct((host.mem_total - host.mem_available) / host.mem_total), "of " + bytes(host.mem_total) + " in use · load " + (host.load || []).map((x) => x.toFixed(2)).join(" ")) : null, - ...((sv && sv.disks) || []).map((d) => tile("Disk space", bytes(d.free) + " free", h("span", { title: d.path }, "of " + bytes(d.total) + ", for " + d.holds.join(", ")))))); + sv && sv.database ? dbTile(sv.database, status) : null, + ...((sv && sv.disks) || []).map(diskTile))); if (sv && sv.named_error) out.push(h("div", { class: "msg info" }, "Client counts need rndc: " + sv.named_error)); // Gauges over time: averages, -1 where there was no sample. const series = (key) => ((gauges.series || []).find((x) => x.key === key) || { values: gauges.times.map(() => -1) }).values.map((v) => (v < 0 ? null : v)); @@ -312,3 +313,25 @@ function disabledPanel() { function exploreSince(since) { return rangeSecs[since] > 7 * 86400 ? "7d" : since; } + +// dbTile is the statistics database's size: what the statistics cost on +// disk (the dashboard's Storage panel has all of Jarl's databases). +function dbTile(db, status) { + const keep = ((status || {}).retention || {}).queries; + const sub = db.query_log_rows != null + ? "the query log holds " + compact(db.query_log_rows) + " answers (" + keep + ")" + : "counters and top lists; no query log"; + return h("a", { class: "card link", href: "#/", title: db.path + " · all of Jarl's databases: Dashboard → Storage" }, + h("div", { class: "label" }, "Statistics database"), h("div", { class: "value" }, bytes(db.bytes)), h("div", { class: "sub" }, sub)); +} + +// diskTile is a disk's free space, with a bar of how full it is. +function diskTile(d) { + const u = d.total ? (d.total - d.free) / d.total : 0; + return h("div", { class: "card", title: d.path }, + h("div", { class: "label" }, "Disk space"), + h("div", { class: "value" }, bytes(d.free) + " free"), + h("div", { class: "disk-bar", role: "img", "aria-label": Math.round(u * 100) + "% used" }, + h("div", { class: "seg other" + (u > 0.9 ? " full" : ""), style: { width: u * 100 + "%" } })), + h("div", { class: "sub" }, h("span", { class: u > 0.9 ? "warn-text" : "" }, Math.round(u * 100) + "% of " + bytes(d.total) + " used"), ", for " + d.holds.join(", "))); +} diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index 60f3449..a060b37 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -613,3 +613,4 @@ table.recent-queries td.oneline-cell { max-width: 40ch; white-space: nowrap; ove .swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; vertical-align: -1px; } .swatch.db { background: var(--accent); } .swatch.other { background: var(--muted); opacity: 0.45; } +.card .disk-bar { margin: 6px 0 4px; height: 8px; }