adam/jarl
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Add optional query statistics

Browse Source 
Off by default (stats in jarl.yaml). When enabled, Jarl adds a loopback
statistics channel and a response log to named's configuration (merged
into existing logging/statistics-channels statements) and switches
response logging with rndc responselog, because the responselog option
only counts when named starts. jarld polls named's counters every minute
and tails the response log (resuming after restarts and across
rotation). It stores aggregates, never individual queries, in a separate
SQLite database: per-minute, hourly and daily counters (queries, result
codes, record types, views, zones, cache hits, upstream latency, DNSSEC)
and hourly/daily top lists (names, clients, NXDOMAIN and SERVFAIL names,
refused clients), each with its own retention. Clients can be anonymized
or excluded.

- Web UI: Statistics page with KPI tiles, queries and failed answers over
  time, a weekday x hour heatmap, top lists and breakdowns; SVG charts
  with hover/keyboard readouts, a table view and light/dark palettes
  validated for colour-vision deficiency.
- CLI: jarl stats status|summary|top|timeline|heatmap|breakdown.
- API: /api/v1/stats/*, behind a new "stats" permission resource.
- jarld brings named's configuration up to date with jarl.yaml at
  startup (engine.SyncConfig), never overwriting hand-edited files.
- resync now refuses to overwrite drifted files without force, as
  documented.
- tmpfiles.d creates /var/log/named (allowed by named's AppArmor profile
  on Debian 13 and Ubuntu).
- The e2e harness runs named with -f like the Debian unit.
- docs/stats.md, plus API, CLI, configuration, permissions and README
  updates.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
abrown38andClaude Opus 5.5 committed 2026-10-07 20:21:34 +00:00
1 parent 0795a1001e
commit 3b7bee67f8
44 files changed
+4163 -42
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/server/h_stats.go
+147
View File
@@ -0,0 +1,147 @@
package server
import (
"net/http"
"strings"
"time"
"jarl/internal/auth"
"jarl/internal/stats"
)
// statsQuerier checks permission and that statistics are enabled.
func (s *Server) statsQuerier(p *auth.Principal) (*stats.Querier, error) {
if err := require(p, auth.Action{Resource: auth.ResStats}); err != nil {
return nil, err
}
if s.statsQ == nil {
return nil, &httpError{http.StatusNotFound, "stats_disabled", "query statistics are disabled; set stats.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/stats.md)"}
}
return s.statsQ, nil
}
func statsSince(r *http.Request) (time.Duration, error) {
d, err := stats.ParseSince(r.URL.Query().Get("since"))
if err != nil {
return 0, badRequest("%v", err)
}
return d, nil
}
func (s *Server) statsStatus(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
if err := require(p, auth.Action{Resource: auth.ResStats}); err != nil {
return err
}
if s.cfg.Stats == nil {
writeJSON(w, http.StatusOK, stats.Status{Enabled: false, Warnings: []string{}})
return nil
}
writeJSON(w, http.StatusOK, s.cfg.Stats.Status())
return nil
}
func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(p)
if err != nil {
return err
}
since, err := statsSince(r)
if err != nil {
return err
}
sum, err := q.Summary(r.Context(), since, queryInt(r, "top", 10))
if err != nil {
return err
}
writeJSON(w, http.StatusOK, sum)
return nil
}
func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(p)
if err != nil {
return err
}
since, err := statsSince(r)
if err != nil {
return err
}
kind := r.URL.Query().Get("kind")
if kind == "" {
kind = stats.KindName
}
limit := min(max(queryInt(r, "limit", 20), 1), 10000)
es, err := q.Top(r.Context(), kind, since, limit)
if err != nil {
return badRequest("%v", err)
}
writeJSON(w, http.StatusOK, es)
return nil
}
func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(p)
if err != nil {
return err
}
since, err := statsSince(r)
if err != nil {
return err
}
var keys []string
for _, k := range r.URL.Query()["key"] {
for _, part := range strings.Split(k, ",") {
if part = strings.TrimSpace(part); part != "" {
keys = append(keys, part)
}
}
}
if len(keys) == 0 {
keys = []string{"queries"}
}
if len(keys) > 50 {
return badRequest("at most 50 series")
}
tl, err := q.Timeline(r.Context(), since, r.URL.Query().Get("step"), keys)
if err != nil {
return badRequest("%v", err)
}
writeJSON(w, http.StatusOK, tl)
return nil
}
func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(p)
if err != nil {
return err
}
since := 28 * 24 * time.Hour
if r.URL.Query().Get("since") != "" {
if since, err = statsSince(r); err != nil {
return err
}
}
hm, err := q.Heatmap(r.Context(), r.URL.Query().Get("key"), since)
if err != nil {
return err
}
writeJSON(w, http.StatusOK, hm)
return nil
}
func (s *Server) statsBreakdown(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(p)
if err != nil {
return err
}
since, err := statsSince(r)
if err != nil {
return err
}
es, err := q.Breakdown(r.Context(), r.URL.Query().Get("group"), since)
if err != nil {
return badRequest("%v", err)
}
writeJSON(w, http.StatusOK, es)
return nil
}
internal/server/routes.go
+7
View File
@@ -100,6 +100,13 @@ func (s *Server) routes() {
m.HandleFunc("POST "+p+"/system/check", s.authed(s.systemCheck))
m.HandleFunc("POST "+p+"/system/resync", s.authed(s.systemResync))
m.HandleFunc("GET "+p+"/audit", s.authed(s.listAudit))
// Query statistics
m.HandleFunc("GET "+p+"/stats/status", s.authed(s.statsStatus))
m.HandleFunc("GET "+p+"/stats/summary", s.authed(s.statsSummary))
m.HandleFunc("GET "+p+"/stats/top", s.authed(s.statsTop))
m.HandleFunc("GET "+p+"/stats/timeline", s.authed(s.statsTimeline))
m.HandleFunc("GET "+p+"/stats/heatmap", s.authed(s.statsHeatmap))
m.HandleFunc("GET "+p+"/stats/breakdown", s.authed(s.statsBreakdown))
m.HandleFunc(p+"/", func(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "not_found", "no such API endpoint: "+r.Method+" "+r.URL.Path)
})
internal/server/server.go
+7
View File
@@ -18,6 +18,7 @@ import (
"jarl/internal/auth"
"jarl/internal/engine"
"jarl/internal/stats"
"jarl/internal/store"
)
@@ -34,6 +35,8 @@ type Config struct {
CookieSecure string
MaxBodyBytes int64
Version string
// Stats is the query statistics collector (nil when disabled).
Stats *stats.Collector
}
// Server is the HTTP server.
@@ -46,6 +49,7 @@ type Server struct {
limiter *rateLimiter
web fs.FS
certs *certReloader
statsQ *stats.Querier
mu sync.Mutex
srvs []*http.Server
@@ -69,6 +73,9 @@ func New(cfg Config, eng *engine.Engine, log *slog.Logger, web fs.FS) *Server {
log = slog.Default()
}
s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web}
if cfg.Stats != nil {
s.statsQ = stats.NewQuerier(cfg.Stats)
}
s.routes()
return s
}
internal/server/server_test.go
+8 -2
View File
@@ -19,6 +19,7 @@ import (
"jarl/internal/engine"
"jarl/internal/model"
"jarl/internal/render"
"jarl/internal/stats"
"jarl/internal/store"
)
@@ -29,7 +30,8 @@ type apiEnv struct {
token string // admin token
}
func newAPIEnv(t *testing.T) *apiEnv {
// newAPIEnv starts an API server; a statistics collector may be given.
func newAPIEnv(t *testing.T, collector ...*stats.Collector) *apiEnv {
t.Helper()
dir := t.TempDir()
st, err := store.Open(filepath.Join(dir, "jarl.db"))
@@ -42,7 +44,11 @@ func newAPIEnv(t *testing.T) *apiEnv {
Paths: render.Paths{ConfigDir: filepath.Join(dir, "etc"), ZoneDir: filepath.Join(dir, "zones"), CacheDir: filepath.Join(dir, "cache")},
StagingDir: filepath.Join(dir, "staging"), NoReload: true, SkipCheck: true,
}, quiet)
srv := New(Config{Version: "test"}, eng, quiet, nil)
cfg := Config{Version: "test"}
if len(collector) > 0 {
cfg.Stats = collector[0]
}
srv := New(cfg, eng, quiet, nil)
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
hash, _ := auth.HashPassword("correct horse battery")
internal/server/stats_test.go
+99
View File
@@ -0,0 +1,99 @@
package server
import (
"context"
"encoding/json"
"io"
"log/slog"
"strings"
"testing"
"time"
"jarl/internal/stats"
)
func TestStatsAPIDisabled(t *testing.T) {
e := newAPIEnv(t)
st := e.do("GET", "/stats/status", e.token, nil, 200)
if st["enabled"] != false {
t.Errorf("status = %v", st)
}
out, status := e.req("GET", "/stats/summary", e.token, nil)
if status != 404 || !json.Valid(out) || !strings.Contains(string(out), "stats_disabled") {
t.Errorf("summary while disabled = %d %s", status, out)
}
}
func TestStatsAPI(t *testing.T) {
ctx := context.Background()
ss, err := stats.OpenStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer ss.Close()
now := time.Now()
minute := now.Unix() - now.Unix()%60 - 120
hour := minute - minute%3600
b := &stats.Batch{}
vals := map[string]int64{"queries": 50, "rcode:NOERROR": 45, "rcode:NXDOMAIN": 5, "qtype:A": 30, "qtype:AAAA": 20, "zone:example.com": 40, "zone:-": 10}
b.AddCounters(stats.ResMinute, minute, vals)
b.AddCounters(stats.ResHour, hour, vals)
b.SetTops(stats.ResHour, hour, stats.KindName, []stats.Entry{{Key: "www.example.com", Count: 30}, {Key: "mail.example.com", Count: 10}})
b.SetTops(stats.ResHour, hour, stats.KindClient, []stats.Entry{{Key: "10.0.0.7", Count: 50}})
if err := ss.Commit(ctx, b); err != nil {
t.Fatal(err)
}
opts := stats.Options{StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour}
c := stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
e := newAPIEnv(t, c)
if st := e.do("GET", "/stats/status", e.token, nil, 200); st["enabled"] != true {
t.Errorf("status = %v", st)
}
sum := e.do("GET", "/stats/summary?since=1h", e.token, nil, 200)
if sum["queries"].(float64) != 50 || sum["resolution"] != "m" {
t.Errorf("summary = %v", sum)
}
tops := e.list("/stats/top?kind=name&since=2h", e.token)
if len(tops) != 2 || tops[0]["key"] != "www.example.com" {
t.Errorf("top names = %v", tops)
}
if b := e.list("/stats/breakdown?group=zone&since=1h", e.token); len(b) != 2 || b[0]["key"] != "example.com" {
t.Errorf("zones = %v", b)
}
tl := e.do("GET", "/stats/timeline?since=1h&key=queries,rcode:*", e.token, nil, 200)
if tl["step"] != "1m" || len(tl["series"].([]any)) != 3 {
t.Errorf("timeline = %v", tl)
}
if hm := e.do("GET", "/stats/heatmap", e.token, nil, 200); hm["key"] != "queries" {
t.Errorf("heatmap = %v", hm)
}
for _, bad := range []string{"/stats/summary?since=forever", "/stats/top?kind=nope", "/stats/breakdown?group=nope", "/stats/timeline?step=3m"} {
if _, status := e.req("GET", bad, e.token, nil); status != 400 {
t.Errorf("%s = %d, want 400", bad, status)
}
}
// Statistics show client addresses: they need the stats grant.
e.do("POST", "/users", e.token, map[string]any{"username": "editor", "password": "editor-password-1", "role": "dns-editor"}, 201)
var editorID int64
for _, u := range e.list("/users", e.token) {
if u["username"] == "editor" {
editorID = int64(u["id"].(float64))
}
}
tok := e.do("POST", "/users/"+itoa(editorID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string)
for _, path := range []string{"/stats/status", "/stats/summary", "/stats/top"} {
if _, status := e.req("GET", path, tok, nil); status != 403 {
t.Errorf("dns-editor %s = %d, want 403", path, status)
}
}
e.do("POST", "/users", e.token, map[string]any{"username": "viewer", "password": "viewer-password-1", "role": "readonly"}, 201)
var viewerID int64
for _, u := range e.list("/users", e.token) {
if u["username"] == "viewer" {
viewerID = int64(u["id"].(float64))
}
}
vtok := e.do("POST", "/users/"+itoa(viewerID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string)
e.do("GET", "/stats/summary", vtok, nil, 200)
}