Explore the statistics: a query log to drill down by client, name, view and more; storage on the dashboard

jarld keeps every answer for a while (stats.retention.queries, default 7 days) in the statistics database: time, client, view, name, type, result and what blocked it. Every client, name, view, record type, result and blocklist on the Statistics page leads to an explore page for it: answers over time, the top values of every other dimension (each a link that narrows further) and the latest answers. GET /stats/explore.

The dashboard shows the size of Jarl's databases (and the query log's rows) and, per disk Jarl uses, how full it is and the databases' share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/h_changes.go

@@ -285,6 +285,7 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa
 		resp.Backup = s.backupStatus(r.Context(), now)
 		resp.TOTP = s.totpOverview(r.Context())
 		resp.Certificates = s.certSummary(r.Context())
+		resp.Storage = s.storage(r.Context())
 	}
 	if p.Can(auth.Action{Resource: auth.ResOptions}) {
 		if h := s.encryptedDNSHealth(); h != nil {

internal/server/h_stats.go

@@ -286,3 +286,59 @@ func (s *Server) statsServer(w http.ResponseWriter, r *http.Request, p *auth.Pri
 	writeJSON(w, http.StatusOK, out)
 	return nil
 }
+
+// statsExplore: GET /stats/explore?client=&name=&domain=&view=&type=&rcode=
+// &blocked=&since=24h explores the query log (docs/stats.md#exploring):
+// totals, a series, the top values of each dimension left open, and the
+// latest answers, with client host names.
+func (s *Server) statsExplore(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(r, p)
+	if err != nil {
+		return err
+	}
+	since, err := statsSince(r)
+	if err != nil {
+		return err
+	}
+	v := r.URL.Query()
+	f := stats.ExploreFilter{Client: v.Get("client"), Name: v.Get("name"), Domain: v.Get("domain"), View: v.Get("view"),
+		Type: v.Get("type"), RCode: v.Get("rcode"), Blocked: v.Get("blocked"), Since: since}
+	ex, err := q.Explore(r.Context(), f, 15, 100)
+	if errors.Is(err, stats.ErrNoQueryLog) {
+		return &httpError{http.StatusNotFound, "query_log_off", err.Error()}
+	}
+	if err != nil {
+		return err
+	}
+	if err := s.nameClients(r, p, ex.By["client"]); err != nil {
+		return err
+	}
+	// The recent answers' clients, named once each.
+	var es []stats.Entry
+	seen := map[string]int{}
+	for _, row := range ex.Recent {
+		if _, ok := seen[row.Client]; !ok {
+			seen[row.Client] = len(es)
+			es = append(es, stats.Entry{Key: row.Client})
+		}
+	}
+	if err := s.nameClients(r, p, es); err != nil {
+		return err
+	}
+	for i := range ex.Recent {
+		ex.Recent[i].ClientName = es[seen[ex.Recent[i].Client]].Name
+	}
+	// The filter's client, named too.
+	out := struct {
+		*stats.Exploration
+		ClientName string `json:"client_name,omitempty"`
+	}{Exploration: ex}
+	if f.Client != "" {
+		one := []stats.Entry{{Key: f.Client}}
+		if err := s.nameClients(r, p, one); err == nil {
+			out.ClientName = one[0].Name
+		}
+	}
+	writeJSON(w, http.StatusOK, out)
+	return nil
+}

internal/server/h_storage.go

@@ -0,0 +1,72 @@
+package server
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+
+	"jarl/internal/api"
+	"jarl/internal/stats"
+)
+
+// storage reports the size of Jarl's databases and how full the disks
+// Jarl uses are (the dashboard's Storage card).
+func (s *Server) storage(ctx context.Context) *api.Storage {
+	out := &api.Storage{Databases: []api.Database{}, Disks: []api.Disk{}}
+	dbFile := func(label, path string) *api.Database {
+		if path == "" || path == ":memory:" {
+			return nil
+		}
+		d := api.Database{Label: label, Path: path}
+		for _, suffix := range []string{"", "-wal", "-shm"} {
+			if fi, err := os.Stat(path + suffix); err == nil {
+				d.Bytes += fi.Size()
+			}
+		}
+		out.Databases = append(out.Databases, d)
+		out.Total += d.Bytes
+		return &out.Databases[len(out.Databases)-1]
+	}
+	labels := []string{}
+	dirs := map[string]string{}
+	add := func(label, dir string) {
+		if dir != "" {
+			labels = append(labels, label)
+			dirs[label] = dir
+		}
+	}
+	const jarlDB, statsDB = "Jarl's database", "statistics database"
+	if p := s.store.Path(); p != "" {
+		dbFile(jarlDB, p)
+		add(jarlDB, filepath.Dir(p))
+	}
+	if s.cfg.Stats != nil {
+		if p := s.cfg.Stats.Store().Path(); p != "" && p != ":memory:" {
+			if d := dbFile(statsDB, p); d != nil && s.cfg.Stats.Options().RetainQueries > 0 {
+				if n, err := s.cfg.Stats.Store().QueryLogSize(ctx); err == nil {
+					d.QueryLogRows = &n
+				}
+			}
+			add(statsDB, filepath.Dir(p))
+		}
+		if lf := s.cfg.Stats.Options().LogFile; lf != "" {
+			add("named's logs", filepath.Dir(lf))
+		}
+	}
+	paths := s.eng.Config().Paths
+	add("zone files", paths.ZoneDir)
+	add("named.conf", paths.ConfigDir)
+	add("Jarl's cache and blocklists", paths.CacheDir)
+	add("backups", s.cfg.BackupDir)
+	for _, d := range stats.DiskUsage(labels, dirs) {
+		disk := api.Disk{Path: d.Path, Holds: d.Holds, Total: d.Total, Free: d.Free}
+		for _, db := range out.Databases {
+			if slices.Contains(d.Holds, db.Label) {
+				disk.Databases += db.Bytes
+			}
+		}
+		out.Disks = append(out.Disks, disk)
+	}
+	return out
+}

internal/server/routes.go

@@ -195,6 +195,7 @@ func (s *Server) routes() {
 	m.HandleFunc("GET "+p+"/stats/heatmap", s.authed(s.statsHeatmap))
 	m.HandleFunc("GET "+p+"/stats/breakdown", s.authed(s.statsBreakdown))
 	m.HandleFunc("GET "+p+"/stats/server", s.authed(s.statsServer))
+	m.HandleFunc("GET "+p+"/stats/explore", s.authed(s.statsExplore))
 	m.HandleFunc(p+"/", func(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusNotFound, "not_found", "no such API endpoint: "+r.Method+" "+r.URL.Path)
 	})

internal/server/stats_test.go

@@ -121,3 +121,58 @@ func TestStatsAPI(t *testing.T) {
 	vtok := e.do("POST", "/users/"+itoa(viewerID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string)
 	e.do("GET", "/stats/summary", vtok, nil, 200)
 }
+
+func TestStatsExploreAPI(t *testing.T) {
+	ctx := context.Background()
+	ss, err := stats.OpenStore(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close()
+	now := time.Now().Unix()
+	b := &stats.Batch{}
+	b.AddQueries([]stats.QueryRow{
+		{Time: now - 60, Client: "10.0.0.7", View: "internal", Name: "www.example.com", Type: "A", RCode: "NOERROR"},
+		{Time: now - 50, Client: "10.0.0.7", View: "internal", Name: "ads.tracker.test", Type: "A", RCode: "NXDOMAIN", Blocked: "hagezi"},
+		{Time: now - 40, Client: "198.51.100.9", View: "external", Name: "nope.example.com", Type: "A", RCode: "NXDOMAIN"},
+	})
+	if err := ss.Commit(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	opts := stats.Options{QueryLog: true, StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour, RetainQueries: 7 * 24 * time.Hour}
+	c := stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	e := newAPIEnv(t, c)
+	e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}, {"name": "laptop", "type": "A", "data": "10.0.0.7"}}}, 201)
+
+	ex := e.do("GET", "/stats/explore?rcode=NXDOMAIN&since=1h", e.token, nil, 200)
+	if ex["total"].(float64) != 2 || ex["blocked"].(float64) != 1 {
+		t.Errorf("NXDOMAIN: %v", ex)
+	}
+	clients := ex["by"].(map[string]any)["client"].([]any)
+	if len(clients) != 2 {
+		t.Fatalf("clients %v", clients)
+	}
+	named := false
+	for _, c := range clients {
+		m := c.(map[string]any)
+		named = named || m["key"] == "10.0.0.7" && m["name"] == "laptop.example.com"
+	}
+	if !named {
+		t.Errorf("client names %v", clients)
+	}
+	ex = e.do("GET", "/stats/explore?client=10.0.0.7", e.token, nil, 200)
+	recent := ex["recent"].([]any)
+	if ex["total"].(float64) != 2 || ex["client_name"] != "laptop.example.com" || len(recent) != 2 || recent[0].(map[string]any)["client_name"] != "laptop.example.com" {
+		t.Errorf("laptop: %v", ex)
+	}
+	if ex := e.do("GET", "/stats/explore?client=10.0.0.7&views=external", e.token, nil, 200); ex["total"].(float64) != 0 {
+		t.Errorf("laptop in external only: %v", ex["total"])
+	}
+
+	// The dashboard reports the databases' size.
+	st := e.do("GET", "/status", e.token, nil, 200)
+	sg, ok := st["storage"].(map[string]any)
+	if !ok || len(sg["databases"].([]any)) == 0 {
+		t.Errorf("storage %v", st["storage"])
+	}
+}