Explore the statistics: a query log to drill down by client, name, view and more; storage on the dashboard

jarld keeps every answer for a while (stats.retention.queries, default 7 days) in the statistics database: time, client, view, name, type, result and what blocked it. Every client, name, view, record type, result and blocklist on the Statistics page leads to an explore page for it: answers over time, the top values of every other dimension (each a link that narrows further) and the latest answers. GET /stats/explore.

The dashboard shows the size of Jarl's databases (and the query log's rows) and, per disk Jarl uses, how full it is and the databases' share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/stats/collector.go

@@ -56,9 +56,12 @@ type Options struct {
 	RetainMinute      time.Duration
 	RetainHour        time.Duration
 	RetainDay         time.Duration
-	Anonymize         bool
-	Exclude           []netip.Prefix
-	Location          *time.Location // for days and log timestamps (default: local time)
+	// RetainQueries is how long the query log keeps every answer (0: no
+	// query log; it needs QueryLog).
+	RetainQueries time.Duration
+	Anonymize     bool
+	Exclude       []netip.Prefix
+	Location      *time.Location // for days and log timestamps (default: local time)
 	// MainConfig is named's main configuration file; it tells named's
 	// process apart from other named processes (for its memory and CPU).
 	MainConfig string
@@ -87,9 +90,10 @@ type Status struct {
 	Warnings          []string   `json:"warnings"`
 	FirstData         *time.Time `json:"first_data,omitempty"`
 	Retention         struct {
-		Minute string `json:"minute"`
-		Hour   string `json:"hour"`
-		Day    string `json:"day"`
+		Minute  string `json:"minute"`
+		Hour    string `json:"hour"`
+		Day     string `json:"day"`
+		Queries string `json:"queries"` // the query log ("" when off)
 	} `json:"retention"`
 }
 
@@ -119,9 +123,11 @@ type Collector struct {
 	recentL  []int64 // response log lines per minute, last 10 minutes
 	linesMin int64
 	ensureAt time.Time
-	// rewritten holds queries named blocked (client#port|name) until their
-	// response line arrives.
-	rewritten map[string]bool
+	// rewritten holds queries named blocked (client#port|name), and what
+	// blocked them, until their response line arrives.
+	rewritten map[string]string
+	// pending are answers for the query log, written at the next flush.
+	pending []QueryRow
 	// viewsSince is set once per-view counters exist (meta views_since).
 	viewsSince   bool
 	viewsSinceAt int64
@@ -137,6 +143,10 @@ type Collector struct {
 // maxKeys bounds the per-hour maps; rarely seen keys are dropped first.
 const maxKeys = 200_000
 
+// maxPending bounds the answers waiting for the query log between flushes
+// (a minute); past it they are counted but not logged.
+const maxPending = 500_000
+
 // NewCollector creates a collector. Run starts it.
 func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger) *Collector {
 	if opts.Location == nil {
@@ -154,6 +164,9 @@ func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger)
 	c.status.Retention.Minute = humanDuration(opts.RetainMinute)
 	c.status.Retention.Hour = humanDuration(opts.RetainHour)
 	c.status.Retention.Day = humanDuration(opts.RetainDay)
+	if opts.QueryLog && opts.RetainQueries > 0 {
+		c.status.Retention.Queries = humanDuration(opts.RetainQueries)
+	}
 	return c
 }
 
@@ -362,7 +375,8 @@ func (c *Collector) handleLine(line string) {
 		}
 		return
 	}
-	blocked := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
+	blockedBy := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
+	blocked := blockedBy != ""
 	if blocked {
 		delete(c.rewritten, ev.Client.String()+"#"+ev.Port+"|"+ev.Name)
 	}
@@ -405,6 +419,9 @@ func (c *Collector) handleLine(line string) {
 	}
 	add(KindName, ev.Name)
 	add(KindClient, client)
+	if c.opts.RetainQueries > 0 && len(c.pending) < maxPending {
+		c.pending = append(c.pending, QueryRow{Time: ev.Time.Unix(), Client: client, View: view, Name: ev.Name, Type: ev.Type, RCode: ev.RCode, Blocked: blockedBy})
+	}
 	switch {
 	case blocked:
 		// Counted by handleRewrite; not a name that doesn't exist.
@@ -445,9 +462,9 @@ func (c *Collector) handleRewrite(rw Rewrite) {
 	if c.rewritten == nil || len(c.rewritten) > 10_000 {
 		// Rewrites whose response line never came (named only logs
 		// responses while response logging is on) don't pile up.
-		c.rewritten = map[string]bool{}
+		c.rewritten = map[string]string{}
 	}
-	c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true
+	c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = policySource(rw.Via)
 	minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time)
 	view := rw.View
 	if view == "" {
@@ -639,6 +656,8 @@ func (c *Collector) flush(ctx context.Context, final bool) {
 		pos, _ := json.Marshal(c.tail.Position())
 		b.SetMeta("tail", string(pos))
 	}
+	b.AddQueries(c.pending)
+	c.pending = nil
 	c.updateWarnings()
 	c.mu.Unlock()
 
@@ -737,6 +756,17 @@ func (c *Collector) prune(ctx context.Context) {
 			c.log.Debug("statistics: pruned", "resolution", res, "rows", n)
 		}
 	}
+	// The query log keeps answers for RetainQueries; with no query log,
+	// none at all (it may have been turned off).
+	keep := c.opts.RetainQueries
+	if !c.opts.QueryLog {
+		keep = 0
+	}
+	if n, err := c.store.PruneQueries(ctx, now-int64(keep.Seconds())); err != nil {
+		c.log.Warn("statistics: pruning the query log failed", "err", err)
+	} else if n > 0 {
+		c.log.Debug("statistics: pruned the query log", "rows", n)
+	}
 }
 
 func humanDuration(d time.Duration) string {

internal/stats/explore.go

@@ -0,0 +1,199 @@
+package stats
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Exploring the query log (docs/stats.md#exploring): every answer of the
+// last days (stats.query_retention), filtered by any mix of client, name,
+// view, type, result and blocking, broken down by each of them.
+
+// ExploreFilter narrows the query log. Empty fields don't filter.
+type ExploreFilter struct {
+	Client string // a client key, as the top lists show it
+	Name   string // exactly this name
+	Domain string // this name or any below it
+	View   string
+	Type   string
+	RCode  string
+	// Blocked is "yes" (blocked by anything), "no", or a blocklist's name
+	// ("rules" for your own rules).
+	Blocked string
+	Since   time.Duration // how far back (default 24h)
+}
+
+// Dimensions are what the query log breaks down by, in the order shown.
+var Dimensions = []string{"client", "name", "view", "type", "rcode", "blocked"}
+
+// Exploration is what the query log says for a filter.
+type Exploration struct {
+	From  time.Time `json:"from"`
+	To    time.Time `json:"to"`
+	Total int64     `json:"total"`
+	// Blocked counts the blocked answers among them.
+	Blocked int64 `json:"blocked"`
+	// Series are the answers (and blocked ones) per Step seconds.
+	Step   int64          `json:"step"`
+	Series []ExplorePoint `json:"series"`
+	// By are the top values of each dimension the filter leaves open.
+	By map[string][]Entry `json:"by"`
+	// Recent are the latest answers.
+	Recent []QueryRow `json:"recent"`
+	// Oldest is the first answer the query log still has.
+	Oldest *time.Time `json:"oldest,omitempty"`
+	// Retention is how long it keeps them.
+	Retention string `json:"retention"`
+}
+
+// ExplorePoint is one step of the series.
+type ExplorePoint struct {
+	Time    int64 `json:"t"`
+	Count   int64 `json:"count"`
+	Blocked int64 `json:"blocked"`
+}
+
+// ErrNoQueryLog is returned when the query log is off.
+var ErrNoQueryLog = fmt.Errorf("the query log is off: it needs stats.query_log and a stats.query_retention above 0 in /etc/jarl/jarl.yaml (docs/stats.md#exploring)")
+
+// where builds the SQL condition and arguments for a filter.
+func (q *Querier) where(f ExploreFilter, from int64) (string, []any) {
+	conds := []string{"ts >= ?"}
+	args := []any{from}
+	eq := func(col, v string) {
+		if v != "" {
+			conds = append(conds, col+" = ?")
+			args = append(args, v)
+		}
+	}
+	eq("client", f.Client)
+	eq("name", strings.ToLower(strings.TrimSuffix(f.Name, ".")))
+	eq("view", f.View)
+	eq("type", strings.ToUpper(f.Type))
+	eq("rcode", strings.ToUpper(f.RCode))
+	if d := strings.ToLower(strings.TrimSuffix(f.Domain, ".")); d != "" {
+		conds = append(conds, "(name = ? OR (length(name) > length(?) AND substr(name, length(name) - length(?)) = ?))")
+		args = append(args, d, d, d, "."+d)
+	}
+	switch f.Blocked {
+	case "":
+	case "yes":
+		conds = append(conds, "blocked != ''")
+	case "no":
+		conds = append(conds, "blocked = ''")
+	default:
+		eq("blocked", f.Blocked)
+	}
+	if len(q.Views) > 0 {
+		marks := strings.TrimSuffix(strings.Repeat("?,", len(q.Views)), ",")
+		conds = append(conds, "view IN ("+marks+")")
+		for _, v := range q.Views {
+			args = append(args, v)
+		}
+	}
+	return strings.Join(conds, " AND "), args
+}
+
+// Explore answers a filter: totals, a series, the top values of each open
+// dimension (topN each) and the latest answers (recent of them).
+func (q *Querier) Explore(ctx context.Context, f ExploreFilter, topN, recent int) (*Exploration, error) {
+	if !q.Opts.QueryLog || q.Opts.RetainQueries <= 0 {
+		return nil, ErrNoQueryLog
+	}
+	now := q.Now()
+	since := f.Since
+	if since <= 0 {
+		since = 24 * time.Hour
+	}
+	if since > q.Opts.RetainQueries {
+		since = q.Opts.RetainQueries
+	}
+	from := now.Add(-since).Unix()
+	out := &Exploration{From: time.Unix(from, 0), To: now, By: map[string][]Entry{}, Series: []ExplorePoint{}, Recent: []QueryRow{},
+		Retention: humanDuration(q.Opts.RetainQueries)}
+	cond, args := q.where(f, from)
+	db := q.Store.db
+
+	if err := db.QueryRowContext(ctx, `SELECT count(*), coalesce(sum(blocked != ''), 0) FROM queries WHERE `+cond, args...).Scan(&out.Total, &out.Blocked); err != nil {
+		return nil, err
+	}
+	// The series: about 100 steps over the range, in whole minutes.
+	step := int64(since.Seconds()) / 100
+	step = max(60, step-step%60)
+	out.Step = step
+	rows, err := db.QueryContext(ctx, `SELECT ts - ts % ? AS t, count(*), sum(blocked != '') FROM queries WHERE `+cond+` GROUP BY t ORDER BY t`, append([]any{step}, args...)...)
+	if err != nil {
+		return nil, err
+	}
+	got := map[int64]ExplorePoint{}
+	for rows.Next() {
+		var p ExplorePoint
+		if err := rows.Scan(&p.Time, &p.Count, &p.Blocked); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		got[p.Time] = p
+	}
+	rows.Close()
+	for t := from - from%step; t <= now.Unix(); t += step {
+		p := got[t]
+		p.Time = t
+		out.Series = append(out.Series, p)
+	}
+
+	open := map[string]bool{"client": f.Client == "", "name": f.Name == "", "view": f.View == "", "type": f.Type == "", "rcode": f.RCode == "", "blocked": f.Blocked == "" || f.Blocked == "yes"}
+	for _, dim := range Dimensions {
+		if !open[dim] {
+			continue
+		}
+		c := cond
+		if dim == "blocked" {
+			c += " AND blocked != ''"
+		}
+		rows, err := db.QueryContext(ctx, `SELECT `+dim+`, count(*) AS n FROM queries WHERE `+c+` GROUP BY `+dim+` ORDER BY n DESC, `+dim+` LIMIT ?`, append(args, topN)...)
+		if err != nil {
+			return nil, err
+		}
+		es := []Entry{}
+		for rows.Next() {
+			var e Entry
+			if err := rows.Scan(&e.Key, &e.Count); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			es = append(es, e)
+		}
+		rows.Close()
+		out.By[dim] = es
+	}
+
+	rows, err = db.QueryContext(ctx, `SELECT ts, client, view, name, type, rcode, blocked FROM queries WHERE `+cond+` ORDER BY ts DESC, rowid DESC LIMIT ?`, append(args, recent)...)
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var r QueryRow
+		if err := rows.Scan(&r.Time, &r.Client, &r.View, &r.Name, &r.Type, &r.RCode, &r.Blocked); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		out.Recent = append(out.Recent, r)
+	}
+	rows.Close()
+
+	var oldest *int64
+	if err := db.QueryRowContext(ctx, `SELECT min(ts) FROM queries`).Scan(&oldest); err == nil && oldest != nil {
+		t := time.Unix(*oldest, 0)
+		out.Oldest = &t
+	}
+	return out, nil
+}
+
+// QueryLogSize returns how many answers the query log has.
+func (s *Store) QueryLogSize(ctx context.Context) (int64, error) {
+	var n int64
+	err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM queries`).Scan(&n)
+	return n, err
+}

internal/stats/explore_test.go

@@ -0,0 +1,98 @@
+package stats
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestExplore(t *testing.T) {
+	ctx := context.Background()
+	loc := time.UTC
+	store, err := OpenStore(filepath.Join(t.TempDir(), "stats.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+	clk := &clock{time.Date(2026, 10, 13, 12, 0, 30, 0, loc)}
+	opts := Options{QueryLog: true, LogFile: "unused", StatisticsChannel: "unused", TopN: 10, RetainMinute: time.Hour, RetainHour: time.Hour, RetainDay: time.Hour,
+		RetainQueries: 7 * 24 * time.Hour, Location: loc}
+	c := NewCollector(store, opts, func() map[string][]string { return nil }, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	c.now = clk.now
+	at := clk.t.Add(-10 * time.Minute)
+	// The phone: lookups, an NXDOMAIN, a blocked tracker.
+	c.HandleLine(logLine(at, "10.0.0.5", "internal", "www.adbraun.lol", "A", "NOERROR"))
+	c.HandleLine(logLine(at, "10.0.0.5", "internal", "nas.adbraun.lol", "AAAA", "NOERROR"))
+	c.HandleLine(logLine(at, "10.0.0.5", "internal", "typo.adbraun.lol", "A", "NXDOMAIN"))
+	c.HandleLine(fmt.Sprintf("%s client @0x7f0000000000 10.0.0.5#5353 (t.tracker.example): view internal: rpz QNAME NXDOMAIN rewrite t.tracker.example/A/IN via t.tracker.example.hagezi.blocklist.jarl.invalid\n", at.Format(logTimeLayout)))
+	c.HandleLine(logLine(at, "10.0.0.5", "internal", "t.tracker.example", "A", "NXDOMAIN"))
+	// A scanner in the external view, NXDOMAIN twice.
+	for i := 0; i < 2; i++ {
+		c.HandleLine(logLine(at, "198.51.100.7", "external", "nope.adbraun.lol", "A", "NXDOMAIN"))
+	}
+	c.flush(ctx, true)
+
+	q := &Querier{Store: store, Opts: opts, Now: clk.now}
+	ex, err := q.Explore(ctx, ExploreFilter{View: "external", RCode: "NXDOMAIN"}, 10, 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ex.Total != 2 || len(ex.By["client"]) != 1 || ex.By["client"][0].Key != "198.51.100.7" || ex.By["view"] != nil {
+		t.Errorf("external NXDOMAIN: %+v", ex)
+	}
+	// What else does the phone do?
+	ex, err = q.Explore(ctx, ExploreFilter{Client: "10.0.0.5"}, 10, 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ex.Total != 4 || ex.Blocked != 1 || len(ex.Recent) != 4 {
+		t.Errorf("phone: total %d blocked %d recent %d", ex.Total, ex.Blocked, len(ex.Recent))
+	}
+	if b := ex.By["blocked"]; len(b) != 1 || b[0].Key != "hagezi" || b[0].Count != 1 {
+		t.Errorf("blocked by %+v", b)
+	}
+	if r := ex.By["rcode"]; len(r) != 2 || r[0].Key != "NOERROR" || r[0].Count != 2 {
+		t.Errorf("rcode %+v", r)
+	}
+	var sum int64
+	for _, p := range ex.Series {
+		sum += p.Count
+	}
+	if sum != 4 || ex.Step != 60*int64(24*60/100) {
+		t.Errorf("series sum %d step %d", sum, ex.Step)
+	}
+	// A domain and what's below it.
+	ex, _ = q.Explore(ctx, ExploreFilter{Domain: "adbraun.lol"}, 10, 50)
+	if ex.Total != 5 {
+		t.Errorf("domain adbraun.lol: %d", ex.Total)
+	}
+	ex, _ = q.Explore(ctx, ExploreFilter{Name: "nas.adbraun.lol", Type: "aaaa"}, 10, 50)
+	if ex.Total != 1 || ex.Recent[0].Client != "10.0.0.5" {
+		t.Errorf("nas AAAA: %+v", ex)
+	}
+	ex, _ = q.Explore(ctx, ExploreFilter{Blocked: "no", Client: "10.0.0.5"}, 10, 50)
+	if ex.Total != 3 {
+		t.Errorf("not blocked: %d", ex.Total)
+	}
+	// Limited to a view (the Statistics page's view switches).
+	ex, _ = q.ForViews([]string{"internal"}).Explore(ctx, ExploreFilter{RCode: "NXDOMAIN"}, 10, 50)
+	if ex.Total != 2 {
+		t.Errorf("internal NXDOMAIN: %d", ex.Total)
+	}
+
+	// Pruning: past the retention they go.
+	clk.t = clk.t.Add(8 * 24 * time.Hour)
+	c.prune(ctx)
+	if n, _ := store.QueryLogSize(ctx); n != 0 {
+		t.Errorf("after pruning: %d rows", n)
+	}
+	// Off: an error that says how to turn it on.
+	q.Opts.RetainQueries = 0
+	if _, err := q.Explore(ctx, ExploreFilter{}, 10, 50); err != ErrNoQueryLog {
+		t.Errorf("off: %v", err)
+	}
+}

internal/stats/store.go

@@ -45,6 +45,21 @@ var schema = []string{
 		PRIMARY KEY (res, ts, kind, key)
 	) WITHOUT ROWID;
 	CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);`,
+	// 2: the query log, every answer for a while (stats.query_retention),
+	// for drilling down: what a client asked, who asked for a name
+	// (docs/stats.md#exploring).
+	`CREATE TABLE queries (
+		ts      INTEGER NOT NULL,
+		client  TEXT NOT NULL,
+		view    TEXT NOT NULL,
+		name    TEXT NOT NULL,
+		type    TEXT NOT NULL,
+		rcode   TEXT NOT NULL,
+		blocked TEXT NOT NULL DEFAULT ''
+	);
+	CREATE INDEX queries_ts ON queries (ts);
+	CREATE INDEX queries_client ON queries (client, ts);
+	CREATE INDEX queries_name ON queries (name, ts);`,
 }
 
 // OpenStore opens (creating if needed) the statistics database. Use
@@ -123,8 +138,25 @@ type Batch struct {
 	counters []counterRow
 	tops     []topsWrite
 	meta     map[string]string
+	queries  []QueryRow
 }
 
+// QueryRow is one answer in the query log.
+type QueryRow struct {
+	Time    int64  `json:"ts"` // Unix seconds
+	Client  string `json:"client"`
+	View    string `json:"view"` // "_default" without views
+	Name    string `json:"name"`
+	Type    string `json:"type"`
+	RCode   string `json:"rcode"`
+	Blocked string `json:"blocked,omitempty"` // the blocklist (or "rules") that blocked it
+	// ClientName is the client's host name (set by the API).
+	ClientName string `json:"client_name,omitempty"`
+}
+
+// AddQueries adds answers to the query log.
+func (b *Batch) AddQueries(rows []QueryRow) { b.queries = append(b.queries, rows...) }
+
 type counterRow struct {
 	res   string
 	ts    int64
@@ -220,9 +252,39 @@ func (s *Store) Commit(ctx context.Context, b *Batch) error {
 			return err
 		}
 	}
+	if len(b.queries) > 0 {
+		st, err := tx.PrepareContext(ctx, `INSERT INTO queries (ts, client, view, name, type, rcode, blocked) VALUES (?, ?, ?, ?, ?, ?, ?)`)
+		if err != nil {
+			return err
+		}
+		for _, q := range b.queries {
+			if _, err := st.ExecContext(ctx, q.Time, q.Client, q.View, q.Name, q.Type, q.RCode, q.Blocked); err != nil {
+				st.Close()
+				return err
+			}
+		}
+		st.Close()
+	}
 	return tx.Commit()
 }
 
+// PruneQueries deletes the query log's answers older than before (Unix
+// time), a slice at a time so the database isn't locked for long.
+func (s *Store) PruneQueries(ctx context.Context, before int64) (int64, error) {
+	var n int64
+	for {
+		r, err := s.db.ExecContext(ctx, `DELETE FROM queries WHERE rowid IN (SELECT rowid FROM queries WHERE ts < ? LIMIT 50000)`, before)
+		if err != nil {
+			return n, err
+		}
+		c, _ := r.RowsAffected()
+		n += c
+		if c < 50000 {
+			return n, nil
+		}
+	}
+}
+
 // Meta returns a stored value ("" if unset).
 func (s *Store) Meta(ctx context.Context, key string) (string, error) {
 	var v string