Add optional query statistics

Off by default (stats in jarl.yaml). When enabled, Jarl adds a loopback
statistics channel and a response log to named's configuration (merged
into existing logging/statistics-channels statements) and switches
response logging with rndc responselog, because the responselog option
only counts when named starts. jarld polls named's counters every minute
and tails the response log (resuming after restarts and across
rotation). It stores aggregates, never individual queries, in a separate
SQLite database: per-minute, hourly and daily counters (queries, result
codes, record types, views, zones, cache hits, upstream latency, DNSSEC)
and hourly/daily top lists (names, clients, NXDOMAIN and SERVFAIL names,
refused clients), each with its own retention. Clients can be anonymized
or excluded.

- Web UI: Statistics page with KPI tiles, queries and failed answers over
  time, a weekday x hour heatmap, top lists and breakdowns; SVG charts
  with hover/keyboard readouts, a table view and light/dark palettes
  validated for colour-vision deficiency.
- CLI: jarl stats status|summary|top|timeline|heatmap|breakdown.
- API: /api/v1/stats/*, behind a new "stats" permission resource.
- jarld brings named's configuration up to date with jarl.yaml at
  startup (engine.SyncConfig), never overwriting hand-edited files.
- resync now refuses to overwrite drifted files without force, as
  documented.
- tmpfiles.d creates /var/log/named (allowed by named's AppArmor profile
  on Debian 13 and Ubuntu).
- The e2e harness runs named with -f like the Debian unit.
- docs/stats.md, plus API, CLI, configuration, permissions and README
  updates.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/render/render.go

@@ -123,12 +123,13 @@ func ZoneFilePath(s *model.State, z *model.Zone, p Paths) string {
 // Header is placed at the top of every generated file.
 const headerText = "Generated by Jarl. DO NOT EDIT: changes made here are overwritten the next time\nchanges are applied. Manage this server through the Jarl web UI, API or CLI."
 
-// Render renders the complete configuration.
-func Render(s *model.State, p Paths) *Output {
+// Render renders the complete configuration. x adds statements that come
+// from jarld's configuration (e.g. for query statistics).
+func Render(s *model.State, p Paths, x Extras) *Output {
 	out := &Output{Spans: map[string][]Span{}, ZoneFiles: map[int64]string{}}
 	keys := renderKeys(s)
 	out.Files = append(out.Files, &File{Path: p.KeysConfig(), Content: []byte(keys), Mode: 0o640, Kind: KindKeys, Secret: true})
-	conf, spans := renderConfig(s, p, out)
+	conf, spans := renderConfig(s, p, x, out)
 	out.Files = append(out.Files, &File{Path: p.MainConfig(), Content: []byte(conf), Mode: 0o640, Kind: KindConfig})
 	out.Spans[p.MainConfig()] = spans
 	for _, z := range s.SortedZones() {
@@ -238,20 +239,43 @@ func normalizeElement(e string) string {
 	return strings.TrimSuffix(strings.TrimSpace(e), ";")
 }
 
-func renderConfig(s *model.State, p Paths, out *Output) (string, []Span) {
+func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Span) {
 	w := newConfWriter()
 	w.write(commentBlock("//", headerText))
 	w.write("\n")
 	w.write(fmt.Sprintf("include %s;\n", namedconf.QuoteString(p.KeysConfig())))
 
+	st := x.Stats
+	logResponses := st != nil && st.LogFile != ""
+	sawStats, sawLogging := false, false
 	if top := s.ScopedOptions(model.ScopeTopLevel, 0); len(top) > 0 {
 		w.write("\n")
 		for _, o := range top {
 			w.begin("option", o.ID, "top-level statement "+o.Name)
-			w.write(optionText(o, 0))
+			switch {
+			case st != nil && o.Name == "statistics-channels" && !sawStats:
+				w.write(st.statsStatement(o))
+				sawStats = true
+			case logResponses && o.Name == "logging" && !sawLogging:
+				w.write(st.loggingStatement(o))
+				sawLogging = true
+			default:
+				w.write(optionText(o, 0))
+			}
 			w.end()
 		}
 	}
+	if st != nil && (!sawStats || logResponses && !sawLogging) {
+		w.write("\n")
+		w.begin("settings", 0, "query statistics settings (stats in jarl.yaml)")
+		if !sawStats {
+			w.write(st.statsStatement(nil))
+		}
+		if logResponses && !sawLogging {
+			w.write(st.loggingStatement(nil))
+		}
+		w.end()
+	}
 
 	if acls := s.SortedACLs(); len(acls) > 0 {
 		w.write("\n")
@@ -276,6 +300,11 @@ func renderConfig(s *model.State, p Paths, out *Output) (string, []Span) {
 		w.write(optionText(o, 1))
 		w.end()
 	}
+	if logResponses && s.FindOption(model.ScopeOptions, 0, "responselog") == nil {
+		w.begin("settings", 0, "query statistics settings (stats in jarl.yaml)")
+		w.write("\t// " + statsComment + "\n\tresponselog yes;\n")
+		w.end()
+	}
 	w.write("};\n")
 
 	views := s.OrderedViews()

internal/render/render_test.go

@@ -90,8 +90,8 @@ func TestRenderDeterministicAndValid(t *testing.T) {
 	}
 	dir := t.TempDir()
 	p := Paths{ConfigDir: filepath.Join(dir, "etc"), ZoneDir: filepath.Join(dir, "zones"), CacheDir: filepath.Join(dir, "cache")}
-	out1 := Render(s, p)
-	out2 := Render(s.Clone(), p)
+	out1 := Render(s, p, Extras{})
+	out2 := Render(s.Clone(), p, Extras{})
 	if len(out1.Files) != len(out2.Files) {
 		t.Fatal("file count differs")
 	}
@@ -143,8 +143,14 @@ func TestRenderDeterministicAndValid(t *testing.T) {
 }
 
 func TestRenderWithNamedCheckconf(t *testing.T) {
+	checkWithNamed(t, sampleState(t), Extras{})
+}
+
+// checkWithNamed renders s and checks the result with named-checkconf -z
+// and named-checkzone.
+func checkWithNamed(t *testing.T, s *model.State, x Extras) *Output {
+	t.Helper()
 	tools := testutil.BindTools(t)
-	s := sampleState(t)
 	dir := t.TempDir()
 	p := Paths{ConfigDir: filepath.Join(dir, "etc"), ZoneDir: filepath.Join(dir, "zones"), CacheDir: filepath.Join(dir, "cache")}
 	// named-checkconf changes into the directory option, so it must exist.
@@ -159,7 +165,7 @@ func TestRenderWithNamedCheckconf(t *testing.T) {
 			z.File = hints
 		}
 	}
-	out := Render(s, p)
+	out := Render(s, p, x)
 	for _, f := range out.Files {
 		if err := os.MkdirAll(filepath.Dir(f.Path), 0o755); err != nil {
 			t.Fatal(err)
@@ -185,6 +191,7 @@ func TestRenderWithNamedCheckconf(t *testing.T) {
 			t.Errorf("named-checkzone %s: %v\n%s", z.Name, err, r.Output)
 		}
 	}
+	return out
 }
 
 func TestZoneContentHash(t *testing.T) {

internal/render/stats.go

@@ -0,0 +1,109 @@
+package render
+
+import (
+	"net/netip"
+	"strconv"
+	"strings"
+
+	"jarl/internal/model"
+	"jarl/internal/namedconf"
+)
+
+// Extras are additions to named's configuration that come from jarld's
+// own configuration (jarl.yaml) rather than from the database.
+type Extras struct {
+	// Stats, when set, makes named publish what Jarl's statistics
+	// collector reads.
+	Stats *StatsHooks
+}
+
+// StatsHooks describe the statistics channel and response log Jarl needs.
+type StatsHooks struct {
+	Channel     netip.AddrPort // statistics-channels address (loopback)
+	LogFile     string         // response log; "" disables response logging
+	LogVersions int
+	LogSize     string
+}
+
+// StatsChannelName is the logging channel Jarl adds for responses.
+const StatsChannelName = "jarl_responses"
+
+const statsComment = "Added by Jarl for query statistics (stats in /etc/jarl/jarl.yaml)."
+
+// statsInet is the statistics-channels clause for the collector.
+func (h *StatsHooks) statsInet() *namedconf.Stmt {
+	addr := h.Channel.Addr().String()
+	return namedconf.NewStmt("inet", addr, "port", strconv.Itoa(int(h.Channel.Port())),
+		"allow", namedconf.NewBlock(namedconf.NewStmt(addr)))
+}
+
+// logChannel is the logging channel the response log is written to.
+func (h *StatsHooks) logChannel() *namedconf.Stmt {
+	return namedconf.NewStmt("channel", StatsChannelName, namedconf.NewBlock(
+		namedconf.NewStmt("file", namedconf.Quoted(h.LogFile), "versions", strconv.Itoa(h.LogVersions), "size", h.LogSize),
+		namedconf.NewStmt("print-time", "yes"),
+		namedconf.NewStmt("severity", "info"),
+	))
+}
+
+// statsStatement returns the statistics-channels statement, merged into the
+// user's own if there is one.
+func (h *StatsHooks) statsStatement(user *model.Option) string {
+	if user != nil {
+		if st, err := namedconf.ParseOptionValue(user.Name, user.Value); err == nil && st.FirstBlock() != nil {
+			b := st.FirstBlock()
+			want := namedconf.CompactStmt(h.statsInet())
+			for _, s := range b.Stmts {
+				if namedconf.CompactStmt(s) == want {
+					return optionText(user, 0)
+				}
+			}
+			b.Stmts = append(b.Stmts, h.statsInet())
+			return commentLines(user.Comment) + commentBlock("//", statsComment) + namedconf.FormatStmt(st, 0)
+		}
+		// Unparsable: validation reports it; leave it alone.
+		return optionText(user, 0)
+	}
+	st := namedconf.NewStmt("statistics-channels", namedconf.NewBlock(h.statsInet()))
+	return commentBlock("//", statsComment) + namedconf.FormatStmt(st, 0)
+}
+
+// loggingStatement returns the logging statement with Jarl's channel and
+// the responses category, merged into the user's own if there is one.
+func (h *StatsHooks) loggingStatement(user *model.Option) string {
+	var st *namedconf.Stmt
+	if user != nil {
+		var err error
+		st, err = namedconf.ParseOptionValue(user.Name, user.Value)
+		if err != nil || st.FirstBlock() == nil {
+			return optionText(user, 0)
+		}
+	} else {
+		st = namedconf.NewStmt("logging", namedconf.NewBlock())
+	}
+	b := st.FirstBlock()
+	// Channels must be defined before the categories that use them.
+	b.Stmts = append([]*namedconf.Stmt{h.logChannel()}, b.Stmts...)
+	merged := false
+	for _, s := range b.Stmts {
+		if s.Keyword() == "category" && strings.EqualFold(s.Arg(0), "responses") && s.FirstBlock() != nil {
+			s.FirstBlock().Stmts = append(s.FirstBlock().Stmts, namedconf.NewStmt(StatsChannelName))
+			merged = true
+		}
+	}
+	if !merged {
+		b.Stmts = append(b.Stmts, namedconf.NewStmt("category", "responses", namedconf.NewBlock(namedconf.NewStmt(StatsChannelName))))
+	}
+	prefix := ""
+	if user != nil {
+		prefix = commentLines(user.Comment)
+	}
+	return prefix + commentBlock("//", statsComment) + namedconf.FormatStmt(st, 0)
+}
+
+func commentLines(c string) string {
+	if c == "" {
+		return ""
+	}
+	return commentBlock("//", c)
+}

internal/render/stats_test.go

@@ -0,0 +1,84 @@
+package render
+
+import (
+	"net/netip"
+	"strings"
+	"testing"
+
+	"jarl/internal/model"
+)
+
+func statsHooks() *StatsHooks {
+	return &StatsHooks{Channel: netip.MustParseAddrPort("127.0.0.1:8058"), LogFile: "/var/log/named/jarl-responses.log", LogVersions: 2, LogSize: "20m"}
+}
+
+func TestStatsHooksAdded(t *testing.T) {
+	s := sampleState(t)
+	// Without a logging statement of its own, Jarl adds one.
+	for id, o := range s.Options {
+		if o.Name == "logging" {
+			delete(s.Options, id)
+		}
+	}
+	p := Paths{ConfigDir: "/etc/bind/jarl", ZoneDir: "/var/lib/bind/jarl", CacheDir: "/var/cache/bind/jarl"}
+	conf := string(Render(s, p, Extras{Stats: statsHooks()}).File(p.MainConfig()).Content)
+	for _, want := range []string{
+		"statistics-channels {\n\tinet 127.0.0.1 port 8058 allow { 127.0.0.1; };\n};",
+		`file "/var/log/named/jarl-responses.log" versions 2 size 20m;`,
+		"\tcategory responses { jarl_responses; };",
+		"\tresponselog yes;\n",
+	} {
+		if !strings.Contains(conf, want) {
+			t.Errorf("named.conf lacks %q:\n%s", want, conf)
+		}
+	}
+	// Counters only: no logging, no responselog.
+	h := statsHooks()
+	h.LogFile = ""
+	conf = string(Render(s, p, Extras{Stats: h}).File(p.MainConfig()).Content)
+	if !strings.Contains(conf, "statistics-channels") || strings.Contains(conf, "responselog") || strings.Contains(conf, "jarl_responses") {
+		t.Errorf("counters-only config:\n%s", conf)
+	}
+	// Disabled: nothing.
+	conf = string(Render(s, p, Extras{}).File(p.MainConfig()).Content)
+	if strings.Contains(conf, "statistics-channels") || strings.Contains(conf, "responselog") {
+		t.Errorf("stats disabled but present:\n%s", conf)
+	}
+}
+
+func TestStatsHooksMerged(t *testing.T) {
+	s := sampleState(t)
+	add := func(scope, name, value string) {
+		id := int64(10000 + len(s.Options))
+		s.Options[id] = &model.Option{ID: id, Scope: scope, Name: name, Value: value, Position: int(id)}
+	}
+	for _, o := range s.Options {
+		if o.Name == "logging" {
+			o.Value = `{ channel default_log { stderr; severity info; }; category default { default_log; }; category responses { default_log; }; }`
+		}
+	}
+	add(model.ScopeTopLevel, "statistics-channels", `{ inet 127.0.0.1 port 8080 allow { 127.0.0.1; }; }`)
+	add(model.ScopeOptions, "responselog", "no")
+	out := checkWithNamed(t, s, Extras{Stats: statsHooks()})
+	var conf string
+	for _, f := range out.Files {
+		if f.Kind == KindConfig {
+			conf = string(f.Content)
+		}
+	}
+	if strings.Count(conf, "logging {") != 1 || strings.Count(conf, "statistics-channels {") != 1 {
+		t.Errorf("statements duplicated instead of merged:\n%s", conf)
+	}
+	for _, want := range []string{"port 8080", "port 8058", "category responses { default_log; jarl_responses; };", "responselog no;"} {
+		if !strings.Contains(conf, want) {
+			t.Errorf("named.conf lacks %q:\n%s", want, conf)
+		}
+	}
+	if strings.Contains(conf, "responselog yes") {
+		t.Errorf("Jarl overrode the user's responselog:\n%s", conf)
+	}
+}
+
+func TestStatsHooksWithNamed(t *testing.T) {
+	checkWithNamed(t, sampleState(t), Extras{Stats: statsHooks()})
+}