Explore the statistics: a query log to drill down by client, name, view and more; storage on the dashboard

jarld keeps every answer for a while (stats.retention.queries, default 7 days) in the statistics database: time, client, view, name, type, result and what blocked it. Every client, name, view, record type, result and blocklist on the Statistics page leads to an explore page for it: answers over time, the top values of every other dimension (each a link that narrows further) and the latest answers. GET /stats/explore.

The dashboard shows the size of Jarl's databases (and the query log's rows) and, per disk Jarl uses, how full it is and the databases' share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/stats/collector.go

@@ -56,9 +56,12 @@ type Options struct {
 	RetainMinute      time.Duration
 	RetainHour        time.Duration
 	RetainDay         time.Duration
-	Anonymize         bool
-	Exclude           []netip.Prefix
-	Location          *time.Location // for days and log timestamps (default: local time)
+	// RetainQueries is how long the query log keeps every answer (0: no
+	// query log; it needs QueryLog).
+	RetainQueries time.Duration
+	Anonymize     bool
+	Exclude       []netip.Prefix
+	Location      *time.Location // for days and log timestamps (default: local time)
 	// MainConfig is named's main configuration file; it tells named's
 	// process apart from other named processes (for its memory and CPU).
 	MainConfig string
@@ -87,9 +90,10 @@ type Status struct {
 	Warnings          []string   `json:"warnings"`
 	FirstData         *time.Time `json:"first_data,omitempty"`
 	Retention         struct {
-		Minute string `json:"minute"`
-		Hour   string `json:"hour"`
-		Day    string `json:"day"`
+		Minute  string `json:"minute"`
+		Hour    string `json:"hour"`
+		Day     string `json:"day"`
+		Queries string `json:"queries"` // the query log ("" when off)
 	} `json:"retention"`
 }
 
@@ -119,9 +123,11 @@ type Collector struct {
 	recentL  []int64 // response log lines per minute, last 10 minutes
 	linesMin int64
 	ensureAt time.Time
-	// rewritten holds queries named blocked (client#port|name) until their
-	// response line arrives.
-	rewritten map[string]bool
+	// rewritten holds queries named blocked (client#port|name), and what
+	// blocked them, until their response line arrives.
+	rewritten map[string]string
+	// pending are answers for the query log, written at the next flush.
+	pending []QueryRow
 	// viewsSince is set once per-view counters exist (meta views_since).
 	viewsSince   bool
 	viewsSinceAt int64
@@ -137,6 +143,10 @@ type Collector struct {
 // maxKeys bounds the per-hour maps; rarely seen keys are dropped first.
 const maxKeys = 200_000
 
+// maxPending bounds the answers waiting for the query log between flushes
+// (a minute); past it they are counted but not logged.
+const maxPending = 500_000
+
 // NewCollector creates a collector. Run starts it.
 func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger) *Collector {
 	if opts.Location == nil {
@@ -154,6 +164,9 @@ func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger)
 	c.status.Retention.Minute = humanDuration(opts.RetainMinute)
 	c.status.Retention.Hour = humanDuration(opts.RetainHour)
 	c.status.Retention.Day = humanDuration(opts.RetainDay)
+	if opts.QueryLog && opts.RetainQueries > 0 {
+		c.status.Retention.Queries = humanDuration(opts.RetainQueries)
+	}
 	return c
 }
 
@@ -362,7 +375,8 @@ func (c *Collector) handleLine(line string) {
 		}
 		return
 	}
-	blocked := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
+	blockedBy := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
+	blocked := blockedBy != ""
 	if blocked {
 		delete(c.rewritten, ev.Client.String()+"#"+ev.Port+"|"+ev.Name)
 	}
@@ -405,6 +419,9 @@ func (c *Collector) handleLine(line string) {
 	}
 	add(KindName, ev.Name)
 	add(KindClient, client)
+	if c.opts.RetainQueries > 0 && len(c.pending) < maxPending {
+		c.pending = append(c.pending, QueryRow{Time: ev.Time.Unix(), Client: client, View: view, Name: ev.Name, Type: ev.Type, RCode: ev.RCode, Blocked: blockedBy})
+	}
 	switch {
 	case blocked:
 		// Counted by handleRewrite; not a name that doesn't exist.
@@ -445,9 +462,9 @@ func (c *Collector) handleRewrite(rw Rewrite) {
 	if c.rewritten == nil || len(c.rewritten) > 10_000 {
 		// Rewrites whose response line never came (named only logs
 		// responses while response logging is on) don't pile up.
-		c.rewritten = map[string]bool{}
+		c.rewritten = map[string]string{}
 	}
-	c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true
+	c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = policySource(rw.Via)
 	minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time)
 	view := rw.View
 	if view == "" {
@@ -639,6 +656,8 @@ func (c *Collector) flush(ctx context.Context, final bool) {
 		pos, _ := json.Marshal(c.tail.Position())
 		b.SetMeta("tail", string(pos))
 	}
+	b.AddQueries(c.pending)
+	c.pending = nil
 	c.updateWarnings()
 	c.mu.Unlock()
 
@@ -737,6 +756,17 @@ func (c *Collector) prune(ctx context.Context) {
 			c.log.Debug("statistics: pruned", "resolution", res, "rows", n)
 		}
 	}
+	// The query log keeps answers for RetainQueries; with no query log,
+	// none at all (it may have been turned off).
+	keep := c.opts.RetainQueries
+	if !c.opts.QueryLog {
+		keep = 0
+	}
+	if n, err := c.store.PruneQueries(ctx, now-int64(keep.Seconds())); err != nil {
+		c.log.Warn("statistics: pruning the query log failed", "err", err)
+	} else if n > 0 {
+		c.log.Debug("statistics: pruned the query log", "rows", n)
+	}
 }
 
 func humanDuration(d time.Duration) string {