Prometheus metrics at /metrics, configured in jarl.yaml

New metrics: section (enabled, listen, allow, token). /metrics is served on
the main listeners, the local socket (root and jarld's user) and optional
metrics-only addresses; a scrape needs an allowed address or the bearer
token. Exports health (named up, problems, drift, backups, blocklists,
DNSSEC phases, token expiry) and, with stats on, named's and Jarl's query
counters since jarld started plus named's process and cache gauges.
docs/metrics.md has example queries and alerts checked with promtool.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/config/config.go

@@ -4,6 +4,7 @@ package config
 import (
 	"bytes"
 	"fmt"
+	"net"
 	"net/netip"
 	"net/url"
 	"os"
@@ -38,6 +39,56 @@ type Config struct {
 	ACME       ACME     `yaml:"acme"`
 	Porkbun    Porkbun  `yaml:"porkbun"`
 	Blocking   Blocking `yaml:"blocking"`
+	Metrics    Metrics  `yaml:"metrics"`
+}
+
+// Metrics configures the Prometheus endpoint /metrics (docs/metrics.md).
+// A scrape is allowed from an address in Allow, or with Token as a bearer
+// token; root and jarld's own user may also scrape on the local socket.
+type Metrics struct {
+	Enabled bool `yaml:"enabled"`
+	// Listen are extra addresses that serve only /metrics, over plain
+	// HTTP. /metrics is always served on the main listeners too.
+	Listen []string `yaml:"listen"`
+	// Allow are the addresses or prefixes that may scrape without a token.
+	Allow []string `yaml:"allow"`
+	// Token, when set, lets any address that sends it as
+	// "Authorization: Bearer TOKEN" scrape.
+	Token string `yaml:"token"`
+}
+
+func (m *Metrics) check() error {
+	if !m.Enabled {
+		return nil
+	}
+	for _, a := range m.Listen {
+		if _, port, err := net.SplitHostPort(a); err != nil || port == "" {
+			return fmt.Errorf("listen %q must be ADDRESS:PORT, e.g. 0.0.0.0:9450", a)
+		}
+	}
+	for _, p := range m.Allow {
+		if _, err := ParsePrefix(p); err != nil {
+			return fmt.Errorf("allow: %v", err)
+		}
+	}
+	if m.Token != "" && (len(m.Token) < 16 || strings.ContainsAny(m.Token, " \t\r\n")) {
+		return fmt.Errorf("token must be at least 16 characters, without spaces (e.g. from 'openssl rand -hex 24')")
+	}
+	if len(m.Allow) == 0 && m.Token == "" {
+		return fmt.Errorf("set allow, token or both, or nobody can scrape")
+	}
+	return nil
+}
+
+// AllowPrefixes returns the parsed allow list.
+func (m *Metrics) AllowPrefixes() []netip.Prefix {
+	var out []netip.Prefix
+	for _, p := range m.Allow {
+		if pf, err := ParsePrefix(p); err == nil {
+			out = append(out, pf.Masked())
+		}
+	}
+	return out
 }
 
 // Blocking configures blocklist downloads (docs/blocking.md). The lists
@@ -323,6 +374,7 @@ func Default() *Config {
 		},
 		ACME:     ACME{HTTPReq: HTTPReq{RequireHTTPS: true}},
 		Blocking: Blocking{Refresh: Duration(24 * time.Hour), MaxDownloadMB: 64, MaxEntries: 1_000_000},
+		Metrics:  Metrics{Allow: []string{"127.0.0.1", "::1"}},
 		Backup:   Backup{Dir: "/var/backups/jarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7, WarnAfter: Duration(2 * 24 * time.Hour)},
 	}
 }
@@ -394,6 +446,9 @@ func (c *Config) check() error {
 	if err := c.Blocking.check(); err != nil {
 		return fmt.Errorf("blocking: %w", err)
 	}
+	if err := c.Metrics.check(); err != nil {
+		return fmt.Errorf("metrics: %w", err)
+	}
 	switch c.Log.Format {
 	case "text", "json":
 	default:
@@ -633,4 +688,23 @@ blocking:
   refresh: 24h           # check each list for changes this often (at least 1h)
   max_download_mb: 64    # refuse larger downloads
   max_entries: 1000000   # refuse lists with more domains
+
+# Prometheus metrics at /metrics (docs/metrics.md): named's counters and
+# Jarl's own (blocking, zones, views), plus health gauges to alert on
+# (named up, validation problems, blocklist errors, backup age, DNSSEC
+# steps, token expiry). Off by default. /metrics is served on the
+# addresses in 'listen' at the top of this file; a scrape is allowed from
+# an address in 'allow' below, or with the token. Query counters need
+# stats.enabled.
+metrics:
+  enabled: false
+  # Extra addresses that serve only /metrics, over plain HTTP, e.g.
+  # ["0.0.0.0:9450"] for a Prometheus server elsewhere on the network.
+  listen: []
+  # Addresses or prefixes that may scrape without a token.
+  allow: ["127.0.0.1", "::1"]
+  # A bearer token (at least 16 characters) that lets any address scrape:
+  # in Prometheus, 'authorization: { credentials: TOKEN }'. Keep this file
+  # readable by root and jarld only when you set it.
+  token: ""
 `

internal/config/config_test.go

@@ -87,3 +87,31 @@ func TestStatsConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestMetricsConfig(t *testing.T) {
+	d := Default()
+	if d.Metrics.Enabled || len(d.Metrics.AllowPrefixes()) != 2 || d.Metrics.Token != "" {
+		t.Errorf("defaults = %+v", d.Metrics)
+	}
+	path := filepath.Join(t.TempDir(), "jarl.yaml")
+	for _, tc := range []struct {
+		yaml string
+		ok   bool
+	}{
+		{"metrics: {enabled: true}", true},
+		{"metrics: {enabled: true, listen: [\"0.0.0.0:9450\", \"[::]:9450\"], allow: [10.0.0.5, 192.168.1.0/24]}", true},
+		{"metrics: {enabled: true, allow: [], token: 0123456789abcdef}", true},
+		{"metrics: {enabled: true, allow: []}", false}, // nobody could scrape
+		{"metrics: {enabled: true, token: short}", false},
+		{"metrics: {enabled: true, token: \"has space 0123456789\"}", false},
+		{"metrics: {enabled: true, listen: [\"9450\"]}", false},
+		{"metrics: {enabled: true, allow: [prometheus.lan]}", false},
+		{"metrics: {enabled: false, listen: [nonsense]}", true}, // only checked when enabled
+	} {
+		os.WriteFile(path, []byte(tc.yaml+"\n"), 0o644)
+		_, err := Load(path, false)
+		if (err == nil) != tc.ok {
+			t.Errorf("%s: err = %v, want ok=%v", tc.yaml, err, tc.ok)
+		}
+	}
+}

internal/server/metrics.go

@@ -0,0 +1,474 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/subtle"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"net"
+	"net/http"
+	"net/netip"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"jarl/internal/dnssec"
+	"jarl/internal/engine"
+	"jarl/internal/model"
+	"jarl/internal/render"
+	"jarl/internal/stats"
+	"jarl/internal/validate"
+)
+
+// Metrics configures the Prometheus endpoint /metrics (docs/metrics.md).
+type Metrics struct {
+	Enabled bool
+	// Listen are extra addresses serving only /metrics (plain HTTP).
+	Listen []string
+	// Allow may scrape without a token; Token, when set, lets anyone who
+	// sends it scrape.
+	Allow []netip.Prefix
+	Token string
+}
+
+// metricsStateTTL is how long the Jarl state part of /metrics is reused:
+// it runs rndc status, validation and the drift check, too much for every
+// scrape when several Prometheus servers scrape often.
+const metricsStateTTL = 15 * time.Second
+
+// metricsCache holds the last Jarl state part of /metrics.
+type metricsCache struct {
+	mu   sync.Mutex
+	at   time.Time
+	body []byte
+}
+
+const metricsContentType = "text/plain; version=0.0.4; charset=utf-8"
+
+// metrics serves GET /metrics in Prometheus's text format.
+func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
+	if !s.cfg.Metrics.Enabled {
+		http.Error(w, "metrics are disabled; set metrics.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/metrics.md)", http.StatusNotFound)
+		return
+	}
+	if code, msg := s.metricsAccess(r); code != 0 {
+		if code == http.StatusUnauthorized {
+			w.Header().Set("WWW-Authenticate", `Bearer realm="jarl metrics"`)
+		}
+		http.Error(w, msg, code)
+		return
+	}
+	var buf bytes.Buffer
+	buf.Write(s.stateMetrics(r.Context()))
+	if c := s.cfg.Stats; c != nil {
+		pw := &promWriter{}
+		statsMetrics(pw, c.Totals(), c.Server(), c.Status())
+		pw.writeTo(&buf)
+	}
+	w.Header().Set("Content-Type", metricsContentType)
+	w.Header().Set("Cache-Control", "no-store")
+	w.Write(buf.Bytes())
+}
+
+// metricsAccess decides whether a scrape is allowed: from an address in
+// metrics.allow, with metrics.token, or on the local socket from root or
+// jarld's own user. It returns 0 when allowed, or the status and message
+// to refuse with.
+func (s *Server) metricsAccess(r *http.Request) (int, string) {
+	m := s.cfg.Metrics
+	ip := s.clientIP(r)
+	if h := r.Header.Get("Authorization"); h != "" {
+		scheme, tok, _ := strings.Cut(h, " ")
+		if m.Token == "" || !strings.EqualFold(scheme, "Bearer") {
+			return http.StatusUnauthorized, "this server takes no token for /metrics (metrics.token is not set)"
+		}
+		if s.limiter.blocked("ip:"+ip, failsPerIP) {
+			return http.StatusTooManyRequests, "too many failed attempts; try again later"
+		}
+		got, want := sha256.Sum256([]byte(strings.TrimSpace(tok))), sha256.Sum256([]byte(m.Token))
+		if subtle.ConstantTimeCompare(got[:], want[:]) != 1 {
+			s.limiter.fail("ip:" + ip)
+			s.log.Warn("metrics: wrong token", "ip", ip)
+			return http.StatusUnauthorized, "wrong token for /metrics"
+		}
+		return 0, ""
+	}
+	if pr := peerOf(r.Context()); pr != nil {
+		if trustedUID(pr.UID) {
+			return 0, ""
+		}
+		return http.StatusForbidden, "on the local socket, only root and jarld's own user may read /metrics without metrics.token"
+	}
+	if a, err := netip.ParseAddr(ip); err == nil {
+		a = a.Unmap().WithZone("")
+		for _, p := range m.Allow {
+			if p.Contains(a) {
+				return 0, ""
+			}
+		}
+	}
+	if m.Token != "" {
+		return http.StatusUnauthorized, "send metrics.token as 'Authorization: Bearer TOKEN', or add " + ip + " to metrics.allow in jarl.yaml"
+	}
+	return http.StatusForbidden, ip + " may not read /metrics: add it to metrics.allow in jarl.yaml (or set metrics.token)"
+}
+
+// serveMetricsListeners serves /metrics alone on metrics.listen.
+func (s *Server) serveMetricsListeners(errc chan error) error {
+	if !s.cfg.Metrics.Enabled || len(s.cfg.Metrics.Listen) == 0 {
+		return nil
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /metrics", s.metrics)
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "this address serves only /metrics", http.StatusNotFound)
+	})
+	h := s.recoverer(s.logRequests(mux))
+	for _, addr := range s.cfg.Metrics.Listen {
+		ln, err := net.Listen("tcp", addr)
+		if err != nil {
+			return fmt.Errorf("metrics.listen %s: %w", addr, err)
+		}
+		srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: time.Minute, IdleTimeout: 2 * time.Minute,
+			ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelWarn)}
+		s.mu.Lock()
+		s.srvs = append(s.srvs, srv)
+		s.mu.Unlock()
+		s.log.Info("serving metrics", "url", "http://"+ln.Addr().String()+"/metrics")
+		go func() {
+			if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+				errc <- err
+			}
+		}()
+	}
+	return nil
+}
+
+// stateMetrics renders Jarl's own state (cached for metricsStateTTL).
+func (s *Server) stateMetrics(ctx context.Context) []byte {
+	c := &s.metricsCache
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := time.Now()
+	if c.body != nil && now.Sub(c.at) < metricsStateTTL {
+		return c.body
+	}
+	pw := &promWriter{}
+	s.collectState(ctx, pw, now)
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	c.body, c.at = buf.Bytes(), now
+	return c.body
+}
+
+func (s *Server) collectState(ctx context.Context, pw *promWriter, now time.Time) {
+	pw.gauge("jarl_build_info", "Jarl's version (always 1).", 1, "version", s.cfg.Version)
+	pw.gauge("jarl_start_time_seconds", "When jarld started (Unix time).", unix(s.started))
+	ok := 1.0
+	defer func() {
+		pw.gauge("jarl_metrics_state_ok", "1 when Jarl's state could be read for this scrape.", ok)
+	}()
+
+	st, err := s.eng.Status(ctx)
+	if err != nil {
+		s.log.Warn("metrics: reading the status failed", "err", err)
+		ok = 0
+		return
+	}
+	pw.gauge("jarl_adopted", "1 when named loads Jarl's configuration.", b2f(st.Adopted))
+	if st.Named != nil || st.NamedError != "" {
+		pw.gauge("jarl_named_up", "1 when named answers rndc status.", b2f(st.Named != nil && st.Named.Running))
+	}
+	if n := st.Named; n != nil {
+		if v := fractions(n.RecursiveClients); len(v) == 3 {
+			pw.gauge("jarl_named_recursive_clients", "Recursive clients named is serving now (rndc status).", v[0])
+			pw.gauge("jarl_named_recursive_clients_soft_limit", "Recursive clients at which named starts dropping the oldest queries.", v[1])
+			pw.gauge("jarl_named_recursive_clients_limit", "Most recursive clients named serves at once (recursive-clients).", v[2])
+		}
+		if v := fractions(n.TCPClients); len(v) == 2 {
+			pw.gauge("jarl_named_tcp_clients", "TCP clients named is serving now (rndc status).", v[0])
+			pw.gauge("jarl_named_tcp_clients_limit", "Most TCP clients named serves at once (tcp-clients).", v[1])
+		}
+	}
+	pw.gauge("jarl_open_changesets", "Changesets with staged changes that are not applied yet.", float64(st.OpenChangesets))
+	if st.LastApplied != nil && st.LastApplied.AppliedAt != nil {
+		pw.gauge("jarl_last_apply_timestamp_seconds", "When the last changeset was applied (Unix time).", unix(*st.LastApplied.AppliedAt))
+	}
+	sev := map[validate.Severity]int{validate.Error: 0, validate.Warning: 0, validate.Info: 0}
+	for _, p := range st.Problems {
+		sev[p.Severity]++
+	}
+	for _, k := range []validate.Severity{validate.Error, validate.Warning, validate.Info} {
+		pw.gauge("jarl_problems", "Validation problems in the current configuration, as on the dashboard.", float64(sev[k]), "severity", string(k))
+	}
+	pw.gauge("jarl_drifted_files", "Files Jarl manages that were changed or removed outside Jarl.", float64(len(st.Drift)))
+	for _, k := range sortedKeys(st.Counts) {
+		pw.gauge("jarl_objects", "Objects in Jarl's configuration.", float64(st.Counts[k]), "kind", k)
+	}
+
+	if s.cfg.BackupDir != "" {
+		bs := s.backupStatus(ctx, now)
+		if bs.LastAt != nil {
+			pw.gauge("jarl_backup_last_success_timestamp_seconds", "When the newest backup was made by 'jarld backup' (Unix time).", unix(*bs.LastAt))
+			pw.gauge("jarl_backup_last_size_bytes", "Size of the newest backup.", float64(bs.LastSize))
+		}
+		if bs.LastErrorAt != nil {
+			pw.gauge("jarl_backup_last_failure_timestamp_seconds", "When a backup last failed, if after the newest backup (Unix time).", unix(*bs.LastErrorAt))
+		}
+		if s.cfg.BackupWarnAfter > 0 {
+			pw.gauge("jarl_backup_stale", "1 when the newest backup is older than backup.warn_after.", b2f(bs.Stale))
+		}
+	}
+
+	state, err := s.store.LoadState(ctx)
+	if err != nil {
+		ok = 0
+		return
+	}
+	if len(state.Blocklists) > 0 || len(state.BlockRules) > 0 {
+		pw.gauge("jarl_blocking_paused", "1 while blocking is paused.", b2f(state.Settings.BlockingPaused()))
+		if t := state.Settings.BlockingPausedUntil; t != nil {
+			pw.gauge("jarl_blocking_paused_until_timestamp_seconds", "When paused blocking resumes (Unix time).", unix(*t))
+		}
+	}
+	statuses, _ := s.store.BlocklistStatuses(ctx)
+	for _, b := range state.SortedBlocklists() {
+		pw.gauge("jarl_blocklist_enabled", "1 for blocklists that are switched on.", b2f(!b.Disabled), "list", b.Name)
+		bs := statuses[b.ID]
+		if bs == nil {
+			continue
+		}
+		pw.gauge("jarl_blocklist_entries", "Domains in the blocklist's last good download.", float64(bs.Entries), "list", b.Name)
+		pw.gauge("jarl_blocklist_failing", "1 when the last download or check of an enabled blocklist failed.", b2f(bs.Error != "" && !b.Disabled), "list", b.Name)
+		if bs.CheckedAt != nil {
+			pw.gauge("jarl_blocklist_last_check_timestamp_seconds", "When the blocklist was last checked for changes (Unix time).", unix(*bs.CheckedAt), "list", b.Name)
+		}
+		if bs.UpdatedAt != nil {
+			pw.gauge("jarl_blocklist_last_change_timestamp_seconds", "When the blocklist's content last changed (Unix time).", unix(*bs.UpdatedAt), "list", b.Name)
+		}
+	}
+
+	zones := make([]*model.Zone, 0, len(state.Zones))
+	for _, z := range state.Zones {
+		if z.DNSSECPolicy != "" && !z.Disabled {
+			zones = append(zones, z)
+		}
+	}
+	slices.SortFunc(zones, func(a, b *model.Zone) int {
+		return strings.Compare(a.Name+" "+state.ViewName(a), b.Name+" "+state.ViewName(b))
+	})
+	for _, z := range zones {
+		keys, err := dnssec.Keys(render.KeyDir(state, z, st.Paths), z.Name)
+		if err != nil {
+			continue
+		}
+		ph := dnssec.Assess(z.DNSSECPolicy, keys)
+		pw.gauge("jarl_dnssec_phase", "The DNSSEC phase of each signed zone (always 1; see docs/dnssec.md). ready: add the DS at the parent; removing_ds: remove it.", 1,
+			"zone", engine.Display(z.Name), "view", state.ViewName(z), "phase", ph.Code)
+	}
+
+	if tokens, err := s.store.ListTokens(ctx, 0); err == nil {
+		for _, t := range tokens {
+			if t.ExpiresAt != nil && t.RevokedAt == nil {
+				pw.gauge("jarl_token_expiry_timestamp_seconds", "When each API token with an expiry date expires (Unix time).", unix(*t.ExpiresAt),
+					"id", strconv.FormatInt(t.ID, 10), "user", t.Username, "token", t.Name)
+			}
+		}
+	}
+}
+
+// counterMetrics maps the statistics counters (docs/stats.md) to metrics:
+// a key "PREFIX:VALUE" becomes metric{label="VALUE"}, a key without a
+// colon a metric without labels.
+var counterMetrics = map[string]struct{ name, label, help string }{
+	"queries":   {"jarl_queries_total", "", "Queries named received (statistics channel)."},
+	"recursion": {"jarl_recursion_queries_total", "", "Queries that caused recursion."},
+	"dropped":   {"jarl_dropped_queries_total", "", "Queries named dropped."},
+	"duplicate": {"jarl_duplicate_queries_total", "", "Duplicate queries named ignored."},
+	"rcode":     {"jarl_responses_by_rcode_total", "rcode", "Responses by response code."},
+	"qtype":     {"jarl_queries_by_type_total", "type", "Queries by record type."},
+	"answer":    {"jarl_answers_total", "source", "Answers that were authoritative or not (recursive)."},
+	"transport": {"jarl_queries_by_transport_total", "transport", "Queries over UDP and TCP."},
+	"ratelimit": {"jarl_ratelimit_responses_total", "action", "Responses rate limiting dropped or slipped (truncated)."},
+	"rtt":       {"jarl_upstream_responses_by_rtt_total", "rtt", "Responses from other servers during recursion, by round-trip time."},
+	"dnssec":    {"jarl_dnssec_validations_total", "result", "DNSSEC validations of recursive answers by result."},
+	"cache":     {"jarl_cache_lookups_total", "result", "Cache lookups that hit or missed."},
+	"responses": {"jarl_logged_responses_total", "", "Responses read from named's response log (without stats.exclude_clients)."},
+	"view":      {"jarl_responses_by_view_total", "view", "Logged responses by view."},
+	"zone":      {"jarl_responses_by_zone_total", "zone", "Logged responses by local zone (\"-\": names in no local zone)."},
+	"blocked":   {"jarl_blocked_queries_total", "", "Queries blocking answered."},
+	"blocklist": {"jarl_blocked_by_list_total", "list", "Blocked queries by blocklist (\"rules\": Jarl's own rules)."},
+}
+
+// statsMetrics adds the query statistics: counters since jarld started,
+// and named's process and server gauges.
+func statsMetrics(pw *promWriter, totals map[string]int64, srv *stats.Server, st stats.Status) {
+	if _, ok := totals["queries"]; !ok {
+		totals["queries"] = 0
+	}
+	for _, k := range sortedKeys(totals) {
+		prefix, value, labeled := strings.Cut(k, ":")
+		m, ok := counterMetrics[prefix]
+		if !ok || labeled != (m.label != "") {
+			continue
+		}
+		if labeled {
+			pw.counter(m.name, m.help, float64(totals[k]), m.label, value)
+		} else {
+			pw.counter(m.name, m.help, float64(totals[k]))
+		}
+	}
+	pw.gauge("jarl_stats_poll_ok", "1 when the last poll of named's statistics channel worked.", b2f(st.PollError == ""))
+	if st.LastPoll != nil {
+		pw.gauge("jarl_stats_last_poll_timestamp_seconds", "When named's statistics channel was last read (Unix time).", unix(*st.LastPoll))
+	}
+	if st.QueryLog {
+		pw.counter("jarl_stats_log_lines_total", "Lines read from named's response log.", float64(st.LinesRead))
+		if st.LastLine != nil {
+			pw.gauge("jarl_stats_last_log_line_timestamp_seconds", "Time of the last response log line read (Unix time).", unix(*st.LastLine))
+		}
+	}
+	pw.gauge("jarl_stats_warnings", "Warnings about statistics collection, as on the Stats page.", float64(len(st.Warnings)))
+	if srv == nil {
+		return
+	}
+	if t, err := time.Parse(time.RFC3339Nano, srv.BootTime); err == nil {
+		pw.gauge("jarl_named_start_time_seconds", "When named started (Unix time).", unix(t))
+	}
+	if t, err := time.Parse(time.RFC3339Nano, srv.ConfigTime); err == nil {
+		pw.gauge("jarl_named_config_time_seconds", "When named last loaded its configuration (Unix time).", unix(t))
+	}
+	if srv.PID != 0 {
+		pw.gauge("jarl_named_resident_memory_bytes", "named's resident memory (/proc).", float64(srv.RSS))
+		pw.gauge("jarl_named_threads", "named's threads (/proc).", float64(srv.Threads))
+		pw.counter("jarl_named_cpu_seconds_total", "CPU time named used since it started (/proc).", srv.CPUSeconds)
+	}
+	if srv.Version != "" {
+		pw.gauge("jarl_named_memory_in_use_bytes", "Memory named counts as in use (statistics channel).", float64(srv.MemInUse))
+		pw.gauge("jarl_named_cache_rrsets", "RRsets in named's caches, all views.", float64(srv.CacheRRsets))
+		pw.gauge("jarl_named_cache_memory_bytes", "Memory named's caches use, all views.", float64(srv.CacheMem))
+		pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsUDP), "proto", "udp")
+		pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsTCP), "proto", "tcp")
+		pw.gauge("jarl_named_recursive_clients_highwater", "Most recursive clients at once since named started.", float64(srv.RecursionHighwater))
+		pw.gauge("jarl_named_tcp_clients_highwater", "Most TCP clients at once since named started.", float64(srv.TCPHighwater))
+	}
+}
+
+// promWriter collects metrics and writes them in Prometheus's text format,
+// each metric's samples together under one HELP and TYPE.
+type promWriter struct {
+	order []string
+	fams  map[string]*promFamily
+}
+
+type promFamily struct {
+	typ, help string
+	lines     []string
+}
+
+func (w *promWriter) gauge(name, help string, v float64, labels ...string) {
+	w.add(name, "gauge", help, v, labels)
+}
+
+func (w *promWriter) counter(name, help string, v float64, labels ...string) {
+	w.add(name, "counter", help, v, labels)
+}
+
+func (w *promWriter) add(name, typ, help string, v float64, labels []string) {
+	if w.fams == nil {
+		w.fams = map[string]*promFamily{}
+	}
+	f := w.fams[name]
+	if f == nil {
+		f = &promFamily{typ: typ, help: help}
+		w.fams[name] = f
+		w.order = append(w.order, name)
+	}
+	var b strings.Builder
+	b.WriteString(name)
+	if len(labels) > 0 {
+		b.WriteByte('{')
+		for i := 0; i+1 < len(labels); i += 2 {
+			if i > 0 {
+				b.WriteByte(',')
+			}
+			b.WriteString(labels[i])
+			b.WriteString(`="`)
+			b.WriteString(labelEscaper.Replace(labels[i+1]))
+			b.WriteByte('"')
+		}
+		b.WriteByte('}')
+	}
+	b.WriteByte(' ')
+	b.WriteString(formatFloat(v))
+	f.lines = append(f.lines, b.String())
+}
+
+func (w *promWriter) writeTo(buf *bytes.Buffer) {
+	for _, name := range w.order {
+		f := w.fams[name]
+		fmt.Fprintf(buf, "# HELP %s %s\n# TYPE %s %s\n", name, helpEscaper.Replace(f.help), name, f.typ)
+		for _, l := range f.lines {
+			buf.WriteString(l)
+			buf.WriteByte('\n')
+		}
+	}
+}
+
+var (
+	labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
+	helpEscaper  = strings.NewReplacer(`\`, `\\`, "\n", `\n`)
+)
+
+func formatFloat(v float64) string {
+	switch {
+	case math.IsInf(v, 1):
+		return "+Inf"
+	case math.IsInf(v, -1):
+		return "-Inf"
+	case math.IsNaN(v):
+		return "NaN"
+	}
+	return strconv.FormatFloat(v, 'f', -1, 64)
+}
+
+func unix(t time.Time) float64 { return float64(t.UnixMilli()) / 1000 }
+
+func b2f(b bool) float64 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// fractions parses rndc status values such as "0/900/1000".
+func fractions(s string) []float64 {
+	if s == "" {
+		return nil
+	}
+	var out []float64
+	for _, p := range strings.Split(s, "/") {
+		v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
+		if err != nil {
+			return nil
+		}
+		out = append(out, v)
+	}
+	return out
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

internal/server/metrics_test.go

@@ -0,0 +1,206 @@
+package server
+
+import (
+	"bytes"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/netip"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"jarl/internal/stats"
+)
+
+// scrape GETs /metrics with an optional bearer token.
+func (e *apiEnv) scrape(token string) (string, int) {
+	e.t.Helper()
+	r, _ := http.NewRequest("GET", e.ts.URL+"/metrics", nil)
+	if token != "" {
+		r.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := http.DefaultClient.Do(r)
+	if err != nil {
+		e.t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return string(b), resp.StatusCode
+}
+
+var (
+	promSampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*"(,[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*")*\})? -?[0-9.eE+-]+$`)
+	promTypeRe   = regexp.MustCompile(`^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (gauge|counter)$`)
+)
+
+// checkExposition checks Prometheus's text format: valid lines, and each
+// metric's samples together under a single TYPE.
+func checkExposition(t *testing.T, body string) {
+	t.Helper()
+	typed := map[string]bool{}
+	cur := ""
+	for _, l := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") {
+		switch {
+		case strings.HasPrefix(l, "# HELP "):
+		case strings.HasPrefix(l, "# TYPE "):
+			m := promTypeRe.FindStringSubmatch(l)
+			if m == nil {
+				t.Errorf("bad TYPE line %q", l)
+				continue
+			}
+			if typed[m[1]] {
+				t.Errorf("metric %s appears twice", m[1])
+			}
+			typed[m[1]], cur = true, m[1]
+		default:
+			if !promSampleRe.MatchString(l) {
+				t.Errorf("bad sample line %q", l)
+			}
+			if name, _, _ := strings.Cut(strings.SplitN(l, " ", 2)[0], "{"); name != cur {
+				t.Errorf("sample %q is not under its TYPE (%s)", l, cur)
+			}
+		}
+	}
+}
+
+func TestMetricsAccess(t *testing.T) {
+	loopback := []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("::1/128")}
+	e := newAPIEnv(t)
+	if _, status := e.scrape(""); status != 404 {
+		t.Errorf("disabled = %d, want 404", status)
+	}
+
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: loopback}})
+	body, status := e.scrape("")
+	if status != 200 {
+		t.Fatalf("allowed scrape = %d: %s", status, body)
+	}
+	checkExposition(t, body)
+	for _, want := range []string{`jarl_build_info{version="test"} 1`, `jarl_problems{severity="error"} 0`, `jarl_objects{kind="zones"} 0`, "jarl_open_changesets 0", "jarl_metrics_state_ok 1"} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "jarl_queries_total") {
+		t.Error("query counters without statistics")
+	}
+	// No token is configured: one that is sent is refused.
+	if _, status := e.scrape("whatever-token-123"); status != 401 {
+		t.Errorf("token without metrics.token = %d, want 401", status)
+	}
+
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}}})
+	if body, status := e.scrape(""); status != 403 || !strings.Contains(body, "metrics.allow") {
+		t.Errorf("not allowed = %d %s", status, body)
+	}
+
+	const token = "0123456789abcdef-metrics"
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Token: token}})
+	if _, status := e.scrape(""); status != 401 {
+		t.Errorf("no token = %d, want 401", status)
+	}
+	if _, status := e.scrape("wrong-token-0123456789"); status != 401 {
+		t.Errorf("wrong token = %d, want 401", status)
+	}
+	if _, status := e.scrape(token); status != 200 {
+		t.Errorf("right token = %d, want 200", status)
+	}
+	// The API's tokens don't count for /metrics.
+	if _, status := e.scrape(e.token); status != 401 {
+		t.Errorf("API token = %d, want 401", status)
+	}
+}
+
+func TestMetricsState(t *testing.T) {
+	e := newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
+	e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
+	e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": "https://lists.example/ads.txt"}, 201)
+	e.srv.metricsCache.body = nil // skip the cache
+	body, _ := e.scrape("")
+	checkExposition(t, body)
+	for _, want := range []string{`jarl_objects{kind="zones"} 1`, `jarl_blocklist_enabled{list="ads"} 1`, "jarl_blocking_paused 0"} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+	// Cached: a change shows after metricsStateTTL.
+	e.do("POST", "/zones", e.token, map[string]any{"name": "example.org", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
+	if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 1`+"\n") {
+		t.Error("state was not cached")
+	}
+	e.srv.metricsCache.at = time.Now().Add(-metricsStateTTL)
+	if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 2`+"\n") {
+		t.Errorf("state not refreshed:\n%s", body)
+	}
+}
+
+func TestMetricsStats(t *testing.T) {
+	ss, err := stats.OpenStore(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close()
+	opts := stats.Options{QueryLog: true, LogFile: "/nonexistent", StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour}
+	c := stats.NewCollector(ss, opts, func() map[string][]string { return map[string][]string{"internal": {"example.com"}} }, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	ts := time.Now().Format("02-Jan-2006 15:04:05.000")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5353 (www.example.com): view internal: response: www.example.com IN A NOERROR 1 0 1 +E(0)K (10.10.0.2)")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: response: ads.example IN A NXDOMAIN 1 0 1 +E(0)K (10.10.0.2)")
+
+	e := newAPIEnvConfig(t, Config{Version: "test", Stats: c, Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
+	body, status := e.scrape("")
+	if status != 200 {
+		t.Fatalf("scrape = %d", status)
+	}
+	checkExposition(t, body)
+	for _, want := range []string{
+		"jarl_queries_total 0", "jarl_logged_responses_total 2", `jarl_responses_by_view_total{view="internal"} 2`,
+		`jarl_responses_by_zone_total{zone="example.com"} 1`, `jarl_responses_by_zone_total{zone="-"} 1`,
+		"jarl_blocked_queries_total 1", `jarl_blocked_by_list_total{list="hagezi"} 1`, "jarl_stats_log_lines_total 2",
+	} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+}
+
+func TestStatsMetricsMapping(t *testing.T) {
+	pw := &promWriter{}
+	totals := map[string]int64{"queries": 10, "rcode:NOERROR": 8, "rcode:NXDOMAIN": 2, "qtype:A": 10, "ratelimit:slipped": 3,
+		"cache:hits": 4, "rtt:<10ms": 5, "unknown:x": 1, "rcode": 7}
+	srv := &stats.Server{BootTime: "2026-10-07T20:00:00.000Z", Version: "9.20.4", PID: 42, RSS: 1 << 20, CPUSeconds: 1.5, SocketsUDP: 4}
+	statsMetrics(pw, totals, srv, stats.Status{})
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	out := buf.String()
+	checkExposition(t, out)
+	for _, want := range []string{
+		"jarl_queries_total 10", `jarl_responses_by_rcode_total{rcode="NXDOMAIN"} 2`, `jarl_ratelimit_responses_total{action="slipped"} 3`,
+		`jarl_upstream_responses_by_rtt_total{rtt="<10ms"} 5`, `jarl_cache_lookups_total{result="hits"} 4`,
+		"jarl_named_start_time_seconds 1791403200", "jarl_named_cpu_seconds_total 1.5", `jarl_named_sockets{proto="udp"} 4`,
+		"jarl_named_resident_memory_bytes 1048576",
+	} {
+		if !strings.Contains(out, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "unknown") || strings.Contains(out, "jarl_responses_by_rcode_total 7") {
+		t.Errorf("unknown keys exported:\n%s", out)
+	}
+}
+
+func TestPromWriterEscaping(t *testing.T) {
+	pw := &promWriter{}
+	pw.gauge("m", "a \\ help\nline", 1, "l", "a\"b\\c\nd")
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	want := "# HELP m a \\\\ help\\nline\n# TYPE m gauge\nm{l=\"a\\\"b\\\\c\\nd\"} 1\n"
+	if buf.String() != want {
+		t.Errorf("got %q, want %q", buf.String(), want)
+	}
+	if got := formatFloat(1759867200.123); got != "1759867200.123" {
+		t.Errorf("formatFloat = %s", got)
+	}
+}

internal/server/routes.go

@@ -22,6 +22,7 @@ func (s *Server) routes() {
 	m.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": s.cfg.Version})
 	})
+	m.HandleFunc("GET /metrics", s.metrics)
 	// Authentication
 	m.HandleFunc("POST "+p+"/auth/login", s.public(s.login))
 	m.HandleFunc("POST "+p+"/auth/login/totp", s.public(s.loginTOTP))

internal/server/server.go

@@ -58,6 +58,8 @@ type Config struct {
 	// name authenticator apps show (h_totp.go).
 	TOTPRequire string
 	TOTPIssuer  string
+	// Metrics configures /metrics (metrics.go).
+	Metrics Metrics
 }
 
 // Server is the HTTP server.
@@ -80,6 +82,10 @@ type Server struct {
 	// nsChecker, when set, replaces the default name server checker
 	// (tests point it at local DNS servers).
 	nsChecker *nscheck.Checker
+	// started is when the server was created; metricsCache holds the Jarl
+	// state part of /metrics.
+	started      time.Time
+	metricsCache metricsCache
 
 	mu   sync.Mutex
 	srvs []*http.Server
@@ -108,7 +114,7 @@ func New(cfg Config, eng *engine.Engine, log *slog.Logger, web fs.FS) *Server {
 	if log == nil {
 		log = slog.Default()
 	}
-	s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web}
+	s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web, started: time.Now()}
 	s.expiryWarn.every = 24 * time.Hour
 	if cfg.Stats != nil {
 		s.statsQ = stats.NewQuerier(cfg.Stats)
@@ -200,6 +206,10 @@ func (s *Server) Serve(ctx context.Context, ready func()) error {
 			}
 		}
 	}
+	if err := s.serveMetricsListeners(errc); err != nil {
+		s.shutdown()
+		return err
+	}
 	s.serveSocket(errc)
 	s.logHTTPReq(tlsCfg != nil)
 	if ready != nil {

internal/stats/collector.go

@@ -114,6 +114,9 @@ type Collector struct {
 	// statistics (server.go).
 	proc   *namedProc
 	server *Server
+	// totals are the counters added up since jarld started (gauges left
+	// out), for Prometheus (Totals).
+	totals map[string]int64
 }
 
 // maxKeys bounds the per-hour maps; rarely seen keys are dropped first.
@@ -125,7 +128,7 @@ func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger)
 		opts.Location = time.Local
 	}
 	c := &Collector{opts: opts, store: store, zones: zones, log: log, poller: NewPoller(opts.StatisticsChannel), now: time.Now,
-		minutes: map[int64]map[string]int64{}, hours: map[int64]map[string]map[string]int64{}, late: map[int64]map[string]map[string]int64{}}
+		minutes: map[int64]map[string]int64{}, totals: map[string]int64{}, hours: map[int64]map[string]map[string]int64{}, late: map[int64]map[string]map[string]int64{}}
 	if opts.QueryLog {
 		c.tail = &Tailer{Path: opts.LogFile}
 	}
@@ -286,6 +289,23 @@ func (c *Collector) count(minute int64, key string, n int64) {
 		c.minutes[minute] = m
 	}
 	m[key] += n
+	if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") {
+		c.totals[key] += n
+	}
+}
+
+// Totals returns every counter added up since the collector started (the
+// keys of docs/stats.md, without gauges). They only grow, so they suit
+// Prometheus counters; named's counters move once a minute, when they are
+// polled.
+func (c *Collector) Totals() map[string]int64 {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make(map[string]int64, len(c.totals))
+	for k, v := range c.totals {
+		out[k] = v
+	}
+	return out
 }
 
 // readLog reads new response log lines.

internal/stats/server.go

@@ -14,7 +14,8 @@ type Server struct {
 	PID        int     `json:"pid,omitempty"`
 	RSS        int64   `json:"rss,omitempty"` // resident memory, bytes
 	Threads    int64   `json:"threads,omitempty"`
-	CPUPercent float64 `json:"cpu_percent"` // of one CPU, over the last minute
+	CPUPercent float64 `json:"cpu_percent"`           // of one CPU, over the last minute
+	CPUSeconds float64 `json:"cpu_seconds,omitempty"` // user + system time since named started
 	// From the statistics channel.
 	MemInUse           int64 `json:"mem_in_use"` // named's own count of the memory it uses
 	CacheRRsets        int64 `json:"cache_rrsets"`
@@ -63,6 +64,7 @@ func (c *Collector) sampleServer(minute int64, now time.Time, snap *Snapshot) {
 		c.proc = nil
 	} else {
 		srv.PID, srv.RSS, srv.Threads = p.PID, p.RSS, p.Threads
+		srv.CPUSeconds = float64(p.CPU) / clockTicks
 		if prev := c.proc; prev.At.Before(now) && !prev.At.IsZero() && p.CPU >= prev.CPU {
 			// Tenths of a percent of one CPU: ticks / clockTicks is CPU seconds.
 			permille := (p.CPU - prev.CPU) * 1000 / clockTicks / max(int64(now.Sub(prev.At)/time.Second), 1)