Prometheus metrics at /metrics, configured in jarl.yaml

New metrics: section (enabled, listen, allow, token). /metrics is served on
the main listeners, the local socket (root and jarld's user) and optional
metrics-only addresses; a scrape needs an allowed address or the bearer
token. Exports health (named up, problems, drift, backups, blocklists,
DNSSEC phases, token expiry) and, with stats on, named's and Jarl's query
counters since jarld started plus named's process and cache gauges.
docs/metrics.md has example queries and alerts checked with promtool.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/metrics.go

@@ -0,0 +1,474 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/subtle"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"net"
+	"net/http"
+	"net/netip"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"jarl/internal/dnssec"
+	"jarl/internal/engine"
+	"jarl/internal/model"
+	"jarl/internal/render"
+	"jarl/internal/stats"
+	"jarl/internal/validate"
+)
+
+// Metrics configures the Prometheus endpoint /metrics (docs/metrics.md).
+type Metrics struct {
+	Enabled bool
+	// Listen are extra addresses serving only /metrics (plain HTTP).
+	Listen []string
+	// Allow may scrape without a token; Token, when set, lets anyone who
+	// sends it scrape.
+	Allow []netip.Prefix
+	Token string
+}
+
+// metricsStateTTL is how long the Jarl state part of /metrics is reused:
+// it runs rndc status, validation and the drift check, too much for every
+// scrape when several Prometheus servers scrape often.
+const metricsStateTTL = 15 * time.Second
+
+// metricsCache holds the last Jarl state part of /metrics.
+type metricsCache struct {
+	mu   sync.Mutex
+	at   time.Time
+	body []byte
+}
+
+const metricsContentType = "text/plain; version=0.0.4; charset=utf-8"
+
+// metrics serves GET /metrics in Prometheus's text format.
+func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
+	if !s.cfg.Metrics.Enabled {
+		http.Error(w, "metrics are disabled; set metrics.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/metrics.md)", http.StatusNotFound)
+		return
+	}
+	if code, msg := s.metricsAccess(r); code != 0 {
+		if code == http.StatusUnauthorized {
+			w.Header().Set("WWW-Authenticate", `Bearer realm="jarl metrics"`)
+		}
+		http.Error(w, msg, code)
+		return
+	}
+	var buf bytes.Buffer
+	buf.Write(s.stateMetrics(r.Context()))
+	if c := s.cfg.Stats; c != nil {
+		pw := &promWriter{}
+		statsMetrics(pw, c.Totals(), c.Server(), c.Status())
+		pw.writeTo(&buf)
+	}
+	w.Header().Set("Content-Type", metricsContentType)
+	w.Header().Set("Cache-Control", "no-store")
+	w.Write(buf.Bytes())
+}
+
+// metricsAccess decides whether a scrape is allowed: from an address in
+// metrics.allow, with metrics.token, or on the local socket from root or
+// jarld's own user. It returns 0 when allowed, or the status and message
+// to refuse with.
+func (s *Server) metricsAccess(r *http.Request) (int, string) {
+	m := s.cfg.Metrics
+	ip := s.clientIP(r)
+	if h := r.Header.Get("Authorization"); h != "" {
+		scheme, tok, _ := strings.Cut(h, " ")
+		if m.Token == "" || !strings.EqualFold(scheme, "Bearer") {
+			return http.StatusUnauthorized, "this server takes no token for /metrics (metrics.token is not set)"
+		}
+		if s.limiter.blocked("ip:"+ip, failsPerIP) {
+			return http.StatusTooManyRequests, "too many failed attempts; try again later"
+		}
+		got, want := sha256.Sum256([]byte(strings.TrimSpace(tok))), sha256.Sum256([]byte(m.Token))
+		if subtle.ConstantTimeCompare(got[:], want[:]) != 1 {
+			s.limiter.fail("ip:" + ip)
+			s.log.Warn("metrics: wrong token", "ip", ip)
+			return http.StatusUnauthorized, "wrong token for /metrics"
+		}
+		return 0, ""
+	}
+	if pr := peerOf(r.Context()); pr != nil {
+		if trustedUID(pr.UID) {
+			return 0, ""
+		}
+		return http.StatusForbidden, "on the local socket, only root and jarld's own user may read /metrics without metrics.token"
+	}
+	if a, err := netip.ParseAddr(ip); err == nil {
+		a = a.Unmap().WithZone("")
+		for _, p := range m.Allow {
+			if p.Contains(a) {
+				return 0, ""
+			}
+		}
+	}
+	if m.Token != "" {
+		return http.StatusUnauthorized, "send metrics.token as 'Authorization: Bearer TOKEN', or add " + ip + " to metrics.allow in jarl.yaml"
+	}
+	return http.StatusForbidden, ip + " may not read /metrics: add it to metrics.allow in jarl.yaml (or set metrics.token)"
+}
+
+// serveMetricsListeners serves /metrics alone on metrics.listen.
+func (s *Server) serveMetricsListeners(errc chan error) error {
+	if !s.cfg.Metrics.Enabled || len(s.cfg.Metrics.Listen) == 0 {
+		return nil
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /metrics", s.metrics)
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "this address serves only /metrics", http.StatusNotFound)
+	})
+	h := s.recoverer(s.logRequests(mux))
+	for _, addr := range s.cfg.Metrics.Listen {
+		ln, err := net.Listen("tcp", addr)
+		if err != nil {
+			return fmt.Errorf("metrics.listen %s: %w", addr, err)
+		}
+		srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: time.Minute, IdleTimeout: 2 * time.Minute,
+			ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelWarn)}
+		s.mu.Lock()
+		s.srvs = append(s.srvs, srv)
+		s.mu.Unlock()
+		s.log.Info("serving metrics", "url", "http://"+ln.Addr().String()+"/metrics")
+		go func() {
+			if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+				errc <- err
+			}
+		}()
+	}
+	return nil
+}
+
+// stateMetrics renders Jarl's own state (cached for metricsStateTTL).
+func (s *Server) stateMetrics(ctx context.Context) []byte {
+	c := &s.metricsCache
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := time.Now()
+	if c.body != nil && now.Sub(c.at) < metricsStateTTL {
+		return c.body
+	}
+	pw := &promWriter{}
+	s.collectState(ctx, pw, now)
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	c.body, c.at = buf.Bytes(), now
+	return c.body
+}
+
+func (s *Server) collectState(ctx context.Context, pw *promWriter, now time.Time) {
+	pw.gauge("jarl_build_info", "Jarl's version (always 1).", 1, "version", s.cfg.Version)
+	pw.gauge("jarl_start_time_seconds", "When jarld started (Unix time).", unix(s.started))
+	ok := 1.0
+	defer func() {
+		pw.gauge("jarl_metrics_state_ok", "1 when Jarl's state could be read for this scrape.", ok)
+	}()
+
+	st, err := s.eng.Status(ctx)
+	if err != nil {
+		s.log.Warn("metrics: reading the status failed", "err", err)
+		ok = 0
+		return
+	}
+	pw.gauge("jarl_adopted", "1 when named loads Jarl's configuration.", b2f(st.Adopted))
+	if st.Named != nil || st.NamedError != "" {
+		pw.gauge("jarl_named_up", "1 when named answers rndc status.", b2f(st.Named != nil && st.Named.Running))
+	}
+	if n := st.Named; n != nil {
+		if v := fractions(n.RecursiveClients); len(v) == 3 {
+			pw.gauge("jarl_named_recursive_clients", "Recursive clients named is serving now (rndc status).", v[0])
+			pw.gauge("jarl_named_recursive_clients_soft_limit", "Recursive clients at which named starts dropping the oldest queries.", v[1])
+			pw.gauge("jarl_named_recursive_clients_limit", "Most recursive clients named serves at once (recursive-clients).", v[2])
+		}
+		if v := fractions(n.TCPClients); len(v) == 2 {
+			pw.gauge("jarl_named_tcp_clients", "TCP clients named is serving now (rndc status).", v[0])
+			pw.gauge("jarl_named_tcp_clients_limit", "Most TCP clients named serves at once (tcp-clients).", v[1])
+		}
+	}
+	pw.gauge("jarl_open_changesets", "Changesets with staged changes that are not applied yet.", float64(st.OpenChangesets))
+	if st.LastApplied != nil && st.LastApplied.AppliedAt != nil {
+		pw.gauge("jarl_last_apply_timestamp_seconds", "When the last changeset was applied (Unix time).", unix(*st.LastApplied.AppliedAt))
+	}
+	sev := map[validate.Severity]int{validate.Error: 0, validate.Warning: 0, validate.Info: 0}
+	for _, p := range st.Problems {
+		sev[p.Severity]++
+	}
+	for _, k := range []validate.Severity{validate.Error, validate.Warning, validate.Info} {
+		pw.gauge("jarl_problems", "Validation problems in the current configuration, as on the dashboard.", float64(sev[k]), "severity", string(k))
+	}
+	pw.gauge("jarl_drifted_files", "Files Jarl manages that were changed or removed outside Jarl.", float64(len(st.Drift)))
+	for _, k := range sortedKeys(st.Counts) {
+		pw.gauge("jarl_objects", "Objects in Jarl's configuration.", float64(st.Counts[k]), "kind", k)
+	}
+
+	if s.cfg.BackupDir != "" {
+		bs := s.backupStatus(ctx, now)
+		if bs.LastAt != nil {
+			pw.gauge("jarl_backup_last_success_timestamp_seconds", "When the newest backup was made by 'jarld backup' (Unix time).", unix(*bs.LastAt))
+			pw.gauge("jarl_backup_last_size_bytes", "Size of the newest backup.", float64(bs.LastSize))
+		}
+		if bs.LastErrorAt != nil {
+			pw.gauge("jarl_backup_last_failure_timestamp_seconds", "When a backup last failed, if after the newest backup (Unix time).", unix(*bs.LastErrorAt))
+		}
+		if s.cfg.BackupWarnAfter > 0 {
+			pw.gauge("jarl_backup_stale", "1 when the newest backup is older than backup.warn_after.", b2f(bs.Stale))
+		}
+	}
+
+	state, err := s.store.LoadState(ctx)
+	if err != nil {
+		ok = 0
+		return
+	}
+	if len(state.Blocklists) > 0 || len(state.BlockRules) > 0 {
+		pw.gauge("jarl_blocking_paused", "1 while blocking is paused.", b2f(state.Settings.BlockingPaused()))
+		if t := state.Settings.BlockingPausedUntil; t != nil {
+			pw.gauge("jarl_blocking_paused_until_timestamp_seconds", "When paused blocking resumes (Unix time).", unix(*t))
+		}
+	}
+	statuses, _ := s.store.BlocklistStatuses(ctx)
+	for _, b := range state.SortedBlocklists() {
+		pw.gauge("jarl_blocklist_enabled", "1 for blocklists that are switched on.", b2f(!b.Disabled), "list", b.Name)
+		bs := statuses[b.ID]
+		if bs == nil {
+			continue
+		}
+		pw.gauge("jarl_blocklist_entries", "Domains in the blocklist's last good download.", float64(bs.Entries), "list", b.Name)
+		pw.gauge("jarl_blocklist_failing", "1 when the last download or check of an enabled blocklist failed.", b2f(bs.Error != "" && !b.Disabled), "list", b.Name)
+		if bs.CheckedAt != nil {
+			pw.gauge("jarl_blocklist_last_check_timestamp_seconds", "When the blocklist was last checked for changes (Unix time).", unix(*bs.CheckedAt), "list", b.Name)
+		}
+		if bs.UpdatedAt != nil {
+			pw.gauge("jarl_blocklist_last_change_timestamp_seconds", "When the blocklist's content last changed (Unix time).", unix(*bs.UpdatedAt), "list", b.Name)
+		}
+	}
+
+	zones := make([]*model.Zone, 0, len(state.Zones))
+	for _, z := range state.Zones {
+		if z.DNSSECPolicy != "" && !z.Disabled {
+			zones = append(zones, z)
+		}
+	}
+	slices.SortFunc(zones, func(a, b *model.Zone) int {
+		return strings.Compare(a.Name+" "+state.ViewName(a), b.Name+" "+state.ViewName(b))
+	})
+	for _, z := range zones {
+		keys, err := dnssec.Keys(render.KeyDir(state, z, st.Paths), z.Name)
+		if err != nil {
+			continue
+		}
+		ph := dnssec.Assess(z.DNSSECPolicy, keys)
+		pw.gauge("jarl_dnssec_phase", "The DNSSEC phase of each signed zone (always 1; see docs/dnssec.md). ready: add the DS at the parent; removing_ds: remove it.", 1,
+			"zone", engine.Display(z.Name), "view", state.ViewName(z), "phase", ph.Code)
+	}
+
+	if tokens, err := s.store.ListTokens(ctx, 0); err == nil {
+		for _, t := range tokens {
+			if t.ExpiresAt != nil && t.RevokedAt == nil {
+				pw.gauge("jarl_token_expiry_timestamp_seconds", "When each API token with an expiry date expires (Unix time).", unix(*t.ExpiresAt),
+					"id", strconv.FormatInt(t.ID, 10), "user", t.Username, "token", t.Name)
+			}
+		}
+	}
+}
+
+// counterMetrics maps the statistics counters (docs/stats.md) to metrics:
+// a key "PREFIX:VALUE" becomes metric{label="VALUE"}, a key without a
+// colon a metric without labels.
+var counterMetrics = map[string]struct{ name, label, help string }{
+	"queries":   {"jarl_queries_total", "", "Queries named received (statistics channel)."},
+	"recursion": {"jarl_recursion_queries_total", "", "Queries that caused recursion."},
+	"dropped":   {"jarl_dropped_queries_total", "", "Queries named dropped."},
+	"duplicate": {"jarl_duplicate_queries_total", "", "Duplicate queries named ignored."},
+	"rcode":     {"jarl_responses_by_rcode_total", "rcode", "Responses by response code."},
+	"qtype":     {"jarl_queries_by_type_total", "type", "Queries by record type."},
+	"answer":    {"jarl_answers_total", "source", "Answers that were authoritative or not (recursive)."},
+	"transport": {"jarl_queries_by_transport_total", "transport", "Queries over UDP and TCP."},
+	"ratelimit": {"jarl_ratelimit_responses_total", "action", "Responses rate limiting dropped or slipped (truncated)."},
+	"rtt":       {"jarl_upstream_responses_by_rtt_total", "rtt", "Responses from other servers during recursion, by round-trip time."},
+	"dnssec":    {"jarl_dnssec_validations_total", "result", "DNSSEC validations of recursive answers by result."},
+	"cache":     {"jarl_cache_lookups_total", "result", "Cache lookups that hit or missed."},
+	"responses": {"jarl_logged_responses_total", "", "Responses read from named's response log (without stats.exclude_clients)."},
+	"view":      {"jarl_responses_by_view_total", "view", "Logged responses by view."},
+	"zone":      {"jarl_responses_by_zone_total", "zone", "Logged responses by local zone (\"-\": names in no local zone)."},
+	"blocked":   {"jarl_blocked_queries_total", "", "Queries blocking answered."},
+	"blocklist": {"jarl_blocked_by_list_total", "list", "Blocked queries by blocklist (\"rules\": Jarl's own rules)."},
+}
+
+// statsMetrics adds the query statistics: counters since jarld started,
+// and named's process and server gauges.
+func statsMetrics(pw *promWriter, totals map[string]int64, srv *stats.Server, st stats.Status) {
+	if _, ok := totals["queries"]; !ok {
+		totals["queries"] = 0
+	}
+	for _, k := range sortedKeys(totals) {
+		prefix, value, labeled := strings.Cut(k, ":")
+		m, ok := counterMetrics[prefix]
+		if !ok || labeled != (m.label != "") {
+			continue
+		}
+		if labeled {
+			pw.counter(m.name, m.help, float64(totals[k]), m.label, value)
+		} else {
+			pw.counter(m.name, m.help, float64(totals[k]))
+		}
+	}
+	pw.gauge("jarl_stats_poll_ok", "1 when the last poll of named's statistics channel worked.", b2f(st.PollError == ""))
+	if st.LastPoll != nil {
+		pw.gauge("jarl_stats_last_poll_timestamp_seconds", "When named's statistics channel was last read (Unix time).", unix(*st.LastPoll))
+	}
+	if st.QueryLog {
+		pw.counter("jarl_stats_log_lines_total", "Lines read from named's response log.", float64(st.LinesRead))
+		if st.LastLine != nil {
+			pw.gauge("jarl_stats_last_log_line_timestamp_seconds", "Time of the last response log line read (Unix time).", unix(*st.LastLine))
+		}
+	}
+	pw.gauge("jarl_stats_warnings", "Warnings about statistics collection, as on the Stats page.", float64(len(st.Warnings)))
+	if srv == nil {
+		return
+	}
+	if t, err := time.Parse(time.RFC3339Nano, srv.BootTime); err == nil {
+		pw.gauge("jarl_named_start_time_seconds", "When named started (Unix time).", unix(t))
+	}
+	if t, err := time.Parse(time.RFC3339Nano, srv.ConfigTime); err == nil {
+		pw.gauge("jarl_named_config_time_seconds", "When named last loaded its configuration (Unix time).", unix(t))
+	}
+	if srv.PID != 0 {
+		pw.gauge("jarl_named_resident_memory_bytes", "named's resident memory (/proc).", float64(srv.RSS))
+		pw.gauge("jarl_named_threads", "named's threads (/proc).", float64(srv.Threads))
+		pw.counter("jarl_named_cpu_seconds_total", "CPU time named used since it started (/proc).", srv.CPUSeconds)
+	}
+	if srv.Version != "" {
+		pw.gauge("jarl_named_memory_in_use_bytes", "Memory named counts as in use (statistics channel).", float64(srv.MemInUse))
+		pw.gauge("jarl_named_cache_rrsets", "RRsets in named's caches, all views.", float64(srv.CacheRRsets))
+		pw.gauge("jarl_named_cache_memory_bytes", "Memory named's caches use, all views.", float64(srv.CacheMem))
+		pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsUDP), "proto", "udp")
+		pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsTCP), "proto", "tcp")
+		pw.gauge("jarl_named_recursive_clients_highwater", "Most recursive clients at once since named started.", float64(srv.RecursionHighwater))
+		pw.gauge("jarl_named_tcp_clients_highwater", "Most TCP clients at once since named started.", float64(srv.TCPHighwater))
+	}
+}
+
+// promWriter collects metrics and writes them in Prometheus's text format,
+// each metric's samples together under one HELP and TYPE.
+type promWriter struct {
+	order []string
+	fams  map[string]*promFamily
+}
+
+type promFamily struct {
+	typ, help string
+	lines     []string
+}
+
+func (w *promWriter) gauge(name, help string, v float64, labels ...string) {
+	w.add(name, "gauge", help, v, labels)
+}
+
+func (w *promWriter) counter(name, help string, v float64, labels ...string) {
+	w.add(name, "counter", help, v, labels)
+}
+
+func (w *promWriter) add(name, typ, help string, v float64, labels []string) {
+	if w.fams == nil {
+		w.fams = map[string]*promFamily{}
+	}
+	f := w.fams[name]
+	if f == nil {
+		f = &promFamily{typ: typ, help: help}
+		w.fams[name] = f
+		w.order = append(w.order, name)
+	}
+	var b strings.Builder
+	b.WriteString(name)
+	if len(labels) > 0 {
+		b.WriteByte('{')
+		for i := 0; i+1 < len(labels); i += 2 {
+			if i > 0 {
+				b.WriteByte(',')
+			}
+			b.WriteString(labels[i])
+			b.WriteString(`="`)
+			b.WriteString(labelEscaper.Replace(labels[i+1]))
+			b.WriteByte('"')
+		}
+		b.WriteByte('}')
+	}
+	b.WriteByte(' ')
+	b.WriteString(formatFloat(v))
+	f.lines = append(f.lines, b.String())
+}
+
+func (w *promWriter) writeTo(buf *bytes.Buffer) {
+	for _, name := range w.order {
+		f := w.fams[name]
+		fmt.Fprintf(buf, "# HELP %s %s\n# TYPE %s %s\n", name, helpEscaper.Replace(f.help), name, f.typ)
+		for _, l := range f.lines {
+			buf.WriteString(l)
+			buf.WriteByte('\n')
+		}
+	}
+}
+
+var (
+	labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
+	helpEscaper  = strings.NewReplacer(`\`, `\\`, "\n", `\n`)
+)
+
+func formatFloat(v float64) string {
+	switch {
+	case math.IsInf(v, 1):
+		return "+Inf"
+	case math.IsInf(v, -1):
+		return "-Inf"
+	case math.IsNaN(v):
+		return "NaN"
+	}
+	return strconv.FormatFloat(v, 'f', -1, 64)
+}
+
+func unix(t time.Time) float64 { return float64(t.UnixMilli()) / 1000 }
+
+func b2f(b bool) float64 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// fractions parses rndc status values such as "0/900/1000".
+func fractions(s string) []float64 {
+	if s == "" {
+		return nil
+	}
+	var out []float64
+	for _, p := range strings.Split(s, "/") {
+		v, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
+		if err != nil {
+			return nil
+		}
+		out = append(out, v)
+	}
+	return out
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

internal/server/metrics_test.go

@@ -0,0 +1,206 @@
+package server
+
+import (
+	"bytes"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/netip"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"jarl/internal/stats"
+)
+
+// scrape GETs /metrics with an optional bearer token.
+func (e *apiEnv) scrape(token string) (string, int) {
+	e.t.Helper()
+	r, _ := http.NewRequest("GET", e.ts.URL+"/metrics", nil)
+	if token != "" {
+		r.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := http.DefaultClient.Do(r)
+	if err != nil {
+		e.t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return string(b), resp.StatusCode
+}
+
+var (
+	promSampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*"(,[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*")*\})? -?[0-9.eE+-]+$`)
+	promTypeRe   = regexp.MustCompile(`^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (gauge|counter)$`)
+)
+
+// checkExposition checks Prometheus's text format: valid lines, and each
+// metric's samples together under a single TYPE.
+func checkExposition(t *testing.T, body string) {
+	t.Helper()
+	typed := map[string]bool{}
+	cur := ""
+	for _, l := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") {
+		switch {
+		case strings.HasPrefix(l, "# HELP "):
+		case strings.HasPrefix(l, "# TYPE "):
+			m := promTypeRe.FindStringSubmatch(l)
+			if m == nil {
+				t.Errorf("bad TYPE line %q", l)
+				continue
+			}
+			if typed[m[1]] {
+				t.Errorf("metric %s appears twice", m[1])
+			}
+			typed[m[1]], cur = true, m[1]
+		default:
+			if !promSampleRe.MatchString(l) {
+				t.Errorf("bad sample line %q", l)
+			}
+			if name, _, _ := strings.Cut(strings.SplitN(l, " ", 2)[0], "{"); name != cur {
+				t.Errorf("sample %q is not under its TYPE (%s)", l, cur)
+			}
+		}
+	}
+}
+
+func TestMetricsAccess(t *testing.T) {
+	loopback := []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("::1/128")}
+	e := newAPIEnv(t)
+	if _, status := e.scrape(""); status != 404 {
+		t.Errorf("disabled = %d, want 404", status)
+	}
+
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: loopback}})
+	body, status := e.scrape("")
+	if status != 200 {
+		t.Fatalf("allowed scrape = %d: %s", status, body)
+	}
+	checkExposition(t, body)
+	for _, want := range []string{`jarl_build_info{version="test"} 1`, `jarl_problems{severity="error"} 0`, `jarl_objects{kind="zones"} 0`, "jarl_open_changesets 0", "jarl_metrics_state_ok 1"} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "jarl_queries_total") {
+		t.Error("query counters without statistics")
+	}
+	// No token is configured: one that is sent is refused.
+	if _, status := e.scrape("whatever-token-123"); status != 401 {
+		t.Errorf("token without metrics.token = %d, want 401", status)
+	}
+
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}}})
+	if body, status := e.scrape(""); status != 403 || !strings.Contains(body, "metrics.allow") {
+		t.Errorf("not allowed = %d %s", status, body)
+	}
+
+	const token = "0123456789abcdef-metrics"
+	e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Token: token}})
+	if _, status := e.scrape(""); status != 401 {
+		t.Errorf("no token = %d, want 401", status)
+	}
+	if _, status := e.scrape("wrong-token-0123456789"); status != 401 {
+		t.Errorf("wrong token = %d, want 401", status)
+	}
+	if _, status := e.scrape(token); status != 200 {
+		t.Errorf("right token = %d, want 200", status)
+	}
+	// The API's tokens don't count for /metrics.
+	if _, status := e.scrape(e.token); status != 401 {
+		t.Errorf("API token = %d, want 401", status)
+	}
+}
+
+func TestMetricsState(t *testing.T) {
+	e := newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
+	e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
+	e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": "https://lists.example/ads.txt"}, 201)
+	e.srv.metricsCache.body = nil // skip the cache
+	body, _ := e.scrape("")
+	checkExposition(t, body)
+	for _, want := range []string{`jarl_objects{kind="zones"} 1`, `jarl_blocklist_enabled{list="ads"} 1`, "jarl_blocking_paused 0"} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+	// Cached: a change shows after metricsStateTTL.
+	e.do("POST", "/zones", e.token, map[string]any{"name": "example.org", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
+	if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 1`+"\n") {
+		t.Error("state was not cached")
+	}
+	e.srv.metricsCache.at = time.Now().Add(-metricsStateTTL)
+	if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 2`+"\n") {
+		t.Errorf("state not refreshed:\n%s", body)
+	}
+}
+
+func TestMetricsStats(t *testing.T) {
+	ss, err := stats.OpenStore(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close()
+	opts := stats.Options{QueryLog: true, LogFile: "/nonexistent", StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour}
+	c := stats.NewCollector(ss, opts, func() map[string][]string { return map[string][]string{"internal": {"example.com"}} }, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	ts := time.Now().Format("02-Jan-2006 15:04:05.000")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5353 (www.example.com): view internal: response: www.example.com IN A NOERROR 1 0 1 +E(0)K (10.10.0.2)")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid")
+	c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: response: ads.example IN A NXDOMAIN 1 0 1 +E(0)K (10.10.0.2)")
+
+	e := newAPIEnvConfig(t, Config{Version: "test", Stats: c, Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
+	body, status := e.scrape("")
+	if status != 200 {
+		t.Fatalf("scrape = %d", status)
+	}
+	checkExposition(t, body)
+	for _, want := range []string{
+		"jarl_queries_total 0", "jarl_logged_responses_total 2", `jarl_responses_by_view_total{view="internal"} 2`,
+		`jarl_responses_by_zone_total{zone="example.com"} 1`, `jarl_responses_by_zone_total{zone="-"} 1`,
+		"jarl_blocked_queries_total 1", `jarl_blocked_by_list_total{list="hagezi"} 1`, "jarl_stats_log_lines_total 2",
+	} {
+		if !strings.Contains(body, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, body)
+		}
+	}
+}
+
+func TestStatsMetricsMapping(t *testing.T) {
+	pw := &promWriter{}
+	totals := map[string]int64{"queries": 10, "rcode:NOERROR": 8, "rcode:NXDOMAIN": 2, "qtype:A": 10, "ratelimit:slipped": 3,
+		"cache:hits": 4, "rtt:<10ms": 5, "unknown:x": 1, "rcode": 7}
+	srv := &stats.Server{BootTime: "2026-10-07T20:00:00.000Z", Version: "9.20.4", PID: 42, RSS: 1 << 20, CPUSeconds: 1.5, SocketsUDP: 4}
+	statsMetrics(pw, totals, srv, stats.Status{})
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	out := buf.String()
+	checkExposition(t, out)
+	for _, want := range []string{
+		"jarl_queries_total 10", `jarl_responses_by_rcode_total{rcode="NXDOMAIN"} 2`, `jarl_ratelimit_responses_total{action="slipped"} 3`,
+		`jarl_upstream_responses_by_rtt_total{rtt="<10ms"} 5`, `jarl_cache_lookups_total{result="hits"} 4`,
+		"jarl_named_start_time_seconds 1791403200", "jarl_named_cpu_seconds_total 1.5", `jarl_named_sockets{proto="udp"} 4`,
+		"jarl_named_resident_memory_bytes 1048576",
+	} {
+		if !strings.Contains(out, want+"\n") {
+			t.Errorf("missing %q in:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "unknown") || strings.Contains(out, "jarl_responses_by_rcode_total 7") {
+		t.Errorf("unknown keys exported:\n%s", out)
+	}
+}
+
+func TestPromWriterEscaping(t *testing.T) {
+	pw := &promWriter{}
+	pw.gauge("m", "a \\ help\nline", 1, "l", "a\"b\\c\nd")
+	var buf bytes.Buffer
+	pw.writeTo(&buf)
+	want := "# HELP m a \\\\ help\\nline\n# TYPE m gauge\nm{l=\"a\\\"b\\\\c\\nd\"} 1\n"
+	if buf.String() != want {
+		t.Errorf("got %q, want %q", buf.String(), want)
+	}
+	if got := formatFloat(1759867200.123); got != "1759867200.123" {
+		t.Errorf("formatFloat = %s", got)
+	}
+}

internal/server/routes.go

@@ -22,6 +22,7 @@ func (s *Server) routes() {
 	m.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": s.cfg.Version})
 	})
+	m.HandleFunc("GET /metrics", s.metrics)
 	// Authentication
 	m.HandleFunc("POST "+p+"/auth/login", s.public(s.login))
 	m.HandleFunc("POST "+p+"/auth/login/totp", s.public(s.loginTOTP))

internal/server/server.go

@@ -58,6 +58,8 @@ type Config struct {
 	// name authenticator apps show (h_totp.go).
 	TOTPRequire string
 	TOTPIssuer  string
+	// Metrics configures /metrics (metrics.go).
+	Metrics Metrics
 }
 
 // Server is the HTTP server.
@@ -80,6 +82,10 @@ type Server struct {
 	// nsChecker, when set, replaces the default name server checker
 	// (tests point it at local DNS servers).
 	nsChecker *nscheck.Checker
+	// started is when the server was created; metricsCache holds the Jarl
+	// state part of /metrics.
+	started      time.Time
+	metricsCache metricsCache
 
 	mu   sync.Mutex
 	srvs []*http.Server
@@ -108,7 +114,7 @@ func New(cfg Config, eng *engine.Engine, log *slog.Logger, web fs.FS) *Server {
 	if log == nil {
 		log = slog.Default()
 	}
-	s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web}
+	s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web, started: time.Now()}
 	s.expiryWarn.every = 24 * time.Hour
 	if cfg.Stats != nil {
 		s.statsQ = stats.NewQuerier(cfg.Stats)
@@ -200,6 +206,10 @@ func (s *Server) Serve(ctx context.Context, ready func()) error {
 			}
 		}
 	}
+	if err := s.serveMetricsListeners(errc); err != nil {
+		s.shutdown()
+		return err
+	}
 	s.serveSocket(errc)
 	s.logHTTPReq(tlsCfg != nil)
 	if ready != nil {