From 12820b17bba1d6706c3e84f92403b0b026416ff2 Mon Sep 17 00:00:00 2001 From: adam Date: Mon, 12 Oct 2026 19:29:33 +0100 Subject: [PATCH] Statistics per view: toggle views on the Statistics page Answers, result codes, record types, zones, blocking, rate limiting and the top lists are also counted per view, from named's response log. The Statistics page has a toggle per view with All and None, remembered per browser; jarl stats --view and ?views= on the API. named's own counters stay for the whole server, and the page says so. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + cmd/jarl/stats.go | 40 ++++-- docs/api.md | 4 + docs/cli.md | 3 + docs/stats.md | 20 +++ internal/server/h_stats.go | 30 ++-- internal/server/stats_test.go | 15 +- internal/stats/collector.go | 95 +++++++++--- internal/stats/query.go | 168 ++++++++++++++++++++-- internal/stats/store.go | 19 +++ internal/stats/views_test.go | 103 +++++++++++++ internal/web/static/assets/pages/stats.js | 76 ++++++++-- internal/web/static/assets/style.css | 7 + 13 files changed, 526 insertions(+), 61 deletions(-) create mode 100644 internal/stats/views_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0175694..beaf44c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,13 @@ All notable changes to Jarl are documented here. The format follows ### Added +- Statistics per view ([docs/stats.md](docs/stats.md#views)): the + Statistics page has a toggle per view, with All and None, remembered in + the browser, to see some views only or several together (linked views, + say). Answers, result codes, record types, zones, blocking, rate + limiting and the top lists follow it; named's own counters stay for the + whole server, and the page says so. `jarl stats --view`, API + `?views=`. Per-view figures start with this version. - DNS over TLS and HTTPS ([docs/encrypted-dns.md](docs/encrypted-dns.md)): `jarl encrypted-dns setup --hostname NAME --address ADDR` (and the Encrypted DNS page) obtains a certificate for the host name if needed, diff --git a/cmd/jarl/stats.go b/cmd/jarl/stats.go index 3f2a5de..226d180 100644 --- a/cmd/jarl/stats.go +++ b/cmd/jarl/stats.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "math" + "net/url" "slices" "strconv" "strings" @@ -21,8 +22,22 @@ func statsCmd() *cobra.Command { Long: `Query statistics collected from named (see docs/stats.md). They must be enabled in /etc/jarl/jarl.yaml (stats.enabled: true). -Ranges are given with --since: 90m, 24h, 7d, 2w. Use -o json or -o yaml to -export the data.`, +Ranges are given with --since: 90m, 24h, 7d, 2w. --view limits the figures +to some views (repeat it, or separate with commas): answers from named's +response log, per view; named's own counters stay for the whole server. +Use -o json or -o yaml to export the data.`, + } + var statsViews []string + cmd.PersistentFlags().StringSliceVar(&statsViews, "view", nil, "only these views (repeat, or comma separated)") + // withViews adds the --view filter to a query. + withViews := func(v url.Values) url.Values { + if len(statsViews) > 0 { + if v == nil { + v = url.Values{} + } + v.Set("views", strings.Join(statsViews, ",")) + } + return v } // Each command has its own --since variable: their defaults differ, and // a shared variable would take the last default for all of them. @@ -77,12 +92,21 @@ export the data.`, return err } var s stats.Summary - if err := c.Get(ctx(), "/stats/summary", q("since", summarySince, "top", strconv.Itoa(topN)), &s); err != nil { + if err := c.Get(ctx(), "/stats/summary", withViews(q("since", summarySince, "top", strconv.Itoa(topN))), &s); err != nil { return err } return output(s, func(w io.Writer) { fmt.Fprintf(w, "Range:\t%s to %s\n", s.From.Local().Format("2006-01-02 15:04"), s.To.Local().Format("2006-01-02 15:04")) - fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + if len(s.FilterViews) > 0 { + since := "" + if s.ViewsSince != nil { + since = ", per view since " + s.ViewsSince.Local().Format("2006-01-02 15:04") + } + fmt.Fprintf(w, "Views:\t%s (from the response log%s; named's own counters are left out)\n", strings.Join(s.FilterViews, ", "), since) + fmt.Fprintf(w, "Answers:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + } else { + fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + } if s.Peak != nil { fmt.Fprintf(w, "Busiest %s:\t%s with %d queries\n", s.Peak.Per, s.Peak.Time.Local().Format("2006-01-02 15:04"), s.Peak.Queries) } @@ -157,7 +181,7 @@ export the data.`, return err } var es []stats.Entry - if err := c.Get(ctx(), "/stats/top", q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit)), &es); err != nil { + if err := c.Get(ctx(), "/stats/top", withViews(q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit))), &es); err != nil { return err } return output(es, func(w io.Writer) { @@ -207,7 +231,7 @@ export the data.`, qv.Add("key", k) } var tl stats.Timeline - if err := c.Get(ctx(), "/stats/timeline", qv, &tl); err != nil { + if err := c.Get(ctx(), "/stats/timeline", withViews(qv), &tl); err != nil { return err } return output(tl, func(w io.Writer) { @@ -256,7 +280,7 @@ export the data.`, return err } var hm stats.Heatmap - if err := c.Get(ctx(), "/stats/heatmap", q("since", heatmapSince, "key", hmKey), &hm); err != nil { + if err := c.Get(ctx(), "/stats/heatmap", withViews(q("since", heatmapSince, "key", hmKey)), &hm); err != nil { return err } return output(hm, func(w io.Writer) { @@ -308,7 +332,7 @@ export the data.`, return err } var es []stats.Entry - if err := c.Get(ctx(), "/stats/breakdown", q("group", args[0], "since", breakdownSince), &es); err != nil { + if err := c.Get(ctx(), "/stats/breakdown", withViews(q("group", args[0], "since", breakdownSince)), &es); err != nil { return err } return output(es, func(w io.Writer) { diff --git a/docs/api.md b/docs/api.md index c2dc004..f048737 100644 --- a/docs/api.md +++ b/docs/api.md @@ -539,6 +539,10 @@ in it and its records. ### Query statistics +Every statistics endpoint below takes `views=internal,vps` to limit the +figures to those views ([stats.md](stats.md#views)); a breakdown that only +exists for the whole server then answers `400 not_per_view`. + Needs `stats:read`. When statistics are disabled, every endpoint but `status` answers 404 `stats_disabled`. See [stats.md](stats.md). diff --git a/docs/cli.md b/docs/cli.md index e4d15b0..59fc7b2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -528,6 +528,9 @@ working. ## Query statistics +`--view internal,vps` (or repeated) limits any `jarl stats` command to those +views ([stats.md](stats.md#views)). + ```sh jarl stats status jarl stats summary --since 7d diff --git a/docs/stats.md b/docs/stats.md index 9c6595e..234db3d 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -184,6 +184,26 @@ jarl stats summary --since 30d -o json # export (also -o yaml) `operator` roles include it, `dns-editor` doesn't. Statistics show client addresses and what they look up, so grant it with care. +## Views + +With views, the Statistics page has a toggle per view (and **All** and +**None**): switch some off to see only the others, for example your +internal and VPS views without the internet's noise on the external one, +or several linked views together. The choice is remembered in your +browser. The CLI takes `--view` on every `jarl stats` command (repeat it, +or separate views with commas), and the API `?views=internal,vps`. + +Only what named's response log says can be split by view: answers (shown +as *answers* instead of queries), result codes, record types, zones, +blocking, rate limiting and the top lists. named's own counters (answer +sources, transport, upstream response times, DNSSEC, the cache and the +server) are for the whole server, and the page says so; asking for them +per view (`jarl stats breakdown answer --view internal`) is refused. + +Per-view figures are kept from the version that added them (Jarl 0.4.1); +the page says since when. They take some more room in the statistics +database, roughly in proportion to the number of views. + ## How data is kept - Minute counters are kept for `retention.minute`, hourly counters and top diff --git a/internal/server/h_stats.go b/internal/server/h_stats.go index bf72f4d..96d3a4e 100644 --- a/internal/server/h_stats.go +++ b/internal/server/h_stats.go @@ -1,6 +1,7 @@ package server import ( + "errors" "net/http" "net/netip" "path/filepath" @@ -14,14 +15,24 @@ import ( "jarl/internal/stats" ) -// statsQuerier checks permission and that statistics are enabled. -func (s *Server) statsQuerier(p *auth.Principal) (*stats.Querier, error) { +// statsQuerier checks permission and that statistics are enabled. With +// ?views=internal,vps the figures are for those views only. +func (s *Server) statsQuerier(r *http.Request, p *auth.Principal) (*stats.Querier, error) { if err := require(p, auth.Action{Resource: auth.ResStats}); err != nil { return nil, err } if s.statsQ == nil { return nil, &httpError{http.StatusNotFound, "stats_disabled", "query statistics are disabled; set stats.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/stats.md)"} } + var views []string + for _, v := range strings.Split(r.URL.Query().Get("views"), ",") { + if v = strings.TrimSpace(v); v != "" { + views = append(views, v) + } + } + if len(views) > 0 { + return s.statsQ.ForViews(views), nil + } return s.statsQ, nil } @@ -46,7 +57,7 @@ func (s *Server) statsStatus(w http.ResponseWriter, r *http.Request, p *auth.Pri } func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -66,7 +77,7 @@ func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Pr } func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -93,7 +104,7 @@ func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Princi } func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -124,7 +135,7 @@ func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.P } func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -143,7 +154,7 @@ func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Pr } func (s *Server) statsBreakdown(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -152,6 +163,9 @@ func (s *Server) statsBreakdown(w http.ResponseWriter, r *http.Request, p *auth. return err } es, err := q.Breakdown(r.Context(), r.URL.Query().Get("group"), since) + if errors.Is(err, stats.ErrNotPerView) { + return &httpError{http.StatusBadRequest, "not_per_view", r.URL.Query().Get("group") + ": " + err.Error()} + } if err != nil { return badRequest("%v", err) } @@ -247,7 +261,7 @@ func clientNames(st *model.State, want map[netip.Addr]bool, readable func(*model // limits (rndc status), and the space left on the file systems Jarl and // named write to. func (s *Server) statsServer(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - if _, err := s.statsQuerier(p); err != nil { + if _, err := s.statsQuerier(r, p); err != nil { return err } out := map[string]any{"server": s.cfg.Stats.Server()} diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go index 685ab9a..57450cc 100644 --- a/internal/server/stats_test.go +++ b/internal/server/stats_test.go @@ -35,7 +35,10 @@ func TestStatsAPI(t *testing.T) { minute := now.Unix() - now.Unix()%60 - 120 hour := minute - minute%3600 b := &stats.Batch{} - vals := map[string]int64{"queries": 50, "rcode:NOERROR": 45, "rcode:NXDOMAIN": 5, "qtype:A": 30, "qtype:AAAA": 20, "zone:example.com": 40, "zone:-": 10} + vals := map[string]int64{"queries": 50, "rcode:NOERROR": 45, "rcode:NXDOMAIN": 5, "qtype:A": 30, "qtype:AAAA": 20, "zone:example.com": 40, "zone:-": 10, + "view:internal": 30, "view:external": 20, stats.ViewKey("internal", "responses"): 30, stats.ViewKey("external", "responses"): 20, + stats.ViewKey("internal", "rcode:NOERROR"): 30, stats.ViewKey("external", "rcode:NOERROR"): 15, stats.ViewKey("external", "rcode:NXDOMAIN"): 5, + "answer:authoritative": 40} b.AddCounters(stats.ResMinute, minute, vals) b.AddCounters(stats.ResHour, hour, vals) b.SetTops(stats.ResHour, hour, stats.KindName, []stats.Entry{{Key: "www.example.com", Count: 30}, {Key: "mail.example.com", Count: 10}}) @@ -47,6 +50,16 @@ func TestStatsAPI(t *testing.T) { c := stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) e := newAPIEnv(t, c) + // Some views only. + if sum := e.do("GET", "/stats/summary?since=1h&views=external", e.token, nil, 200); sum["queries"].(float64) != 20 || len(sum["rcodes"].([]any)) != 2 || sum["filter_views"] == nil { + t.Errorf("external summary = %v", sum) + } + if b := e.list("/stats/breakdown?group=rcode&since=1h&views=internal,external", e.token); len(b) != 2 || b[0]["count"].(float64) != 45 { + t.Errorf("rcodes of both views = %v", b) + } + if out, status := e.req("GET", "/stats/breakdown?group=answer&since=1h&views=internal", e.token, nil); status != 400 || !strings.Contains(string(out), "not_per_view") { + t.Errorf("answers per view = %d %s", status, out) + } if st := e.do("GET", "/stats/status", e.token, nil, 200); st["enabled"] != true { t.Errorf("status = %v", st) } diff --git a/internal/stats/collector.go b/internal/stats/collector.go index 775fd83..d052fb4 100644 --- a/internal/stats/collector.go +++ b/internal/stats/collector.go @@ -33,6 +33,13 @@ const ( KindRateLimitedName = "ratelimited_name" ) +// ViewKey is the key of a counter for one view: per-view copies of the +// counters that come from named's response log (docs/stats.md). +func ViewKey(view, key string) string { return "@" + view + "/" + key } + +// ViewKind is the kind of a top list for one view. +func ViewKind(kind, view string) string { return kind + "@" + view } + // Kinds lists the top-list kinds. var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused, KindBlocked, KindBlockedClient, KindRateLimited, KindRateLimitedName} @@ -115,6 +122,9 @@ type Collector struct { // rewritten holds queries named blocked (client#port|name) until their // response line arrives. rewritten map[string]bool + // viewsSince is set once per-view counters exist (meta views_since). + viewsSince bool + viewsSinceAt int64 // proc is the last sample of named's process, server the latest server // statistics (server.go). proc *namedProc @@ -212,6 +222,9 @@ func (c *Collector) start(ctx context.Context) { } } } + if v, _ := c.store.Meta(ctx, "views_since"); v != "" { + c.viewsSince = true + } if v, _ := c.store.Meta(ctx, "first_data"); v != "" { if n, err := strconv.ParseInt(v, 10, 64); err == nil { t := time.Unix(n, 0) @@ -227,7 +240,8 @@ func (c *Collector) start(ctx context.Context) { if last, err := strconv.ParseInt(merged, 10, 64); err == nil { from := max(last+3600, cur-48*3600) for h := from; h < cur; h += 3600 { - for _, k := range Kinds { + kinds, _ := c.store.KindsAt(ctx, ResHour, h) + for _, k := range kinds { if es, err := c.store.TopsAt(ctx, ResHour, h, k); err == nil && len(es) > 0 { b.AddTops(ResDay, c.dayOf(time.Unix(h, 0)), k, es, 3*c.opts.TopN) } @@ -235,7 +249,8 @@ func (c *Collector) start(ctx context.Context) { } } b.SetMeta("merged_through", strconv.FormatInt(cur-3600, 10)) - for _, k := range Kinds { + curKinds, _ := c.store.KindsAt(ctx, ResHour, cur) + for _, k := range curKinds { es, err := c.store.TopsAt(ctx, ResHour, cur, k) if err != nil || len(es) == 0 { continue @@ -294,7 +309,7 @@ func (c *Collector) count(minute int64, key string, n int64) { c.minutes[minute] = m } m[key] += n - if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") { + if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") && !strings.HasPrefix(key, "@") { c.totals[key] += n } } @@ -365,33 +380,55 @@ func (c *Collector) handleLine(line string) { if view == "" { view = "_default" } + zone := c.zoneOf(ev.Name, ev.View) c.count(minute, "responses", 1) c.count(minute, "view:"+view, 1) - c.count(minute, "zone:"+c.zoneOf(ev.Name, ev.View), 1) + c.count(minute, "zone:"+zone, 1) + // The same, per view, for the statistics of chosen views. + c.countView(minute, view, "responses", 1) + c.countView(minute, view, "zone:"+zone, 1) + c.countView(minute, view, "rcode:"+ev.RCode, 1) + c.countView(minute, view, "qtype:"+ev.Type, 1) client := clientKey(ev.Client, c.opts.Anonymize) hours := c.hours if hour < c.liveHour { hours = c.late } hours[hour] = c.hourMaps(hours, hour) - tops := hours[hour] - tops[KindName][ev.Name]++ - tops[KindClient][client]++ - switch { - case blocked: - // Counted by handleRewrite; not a name that doesn't exist. - case ev.RCode == "NXDOMAIN": - tops[KindNXDomain][ev.Name]++ - case ev.RCode == "SERVFAIL": - tops[KindServFail][ev.Name]++ - case ev.RCode == "REFUSED": - tops[KindRefused][client]++ - } - for _, m := range tops { + add := func(kind, key string) { + hours[hour][kind][key]++ + m := c.topsMap(hours, hour, ViewKind(kind, view)) + m[key]++ if len(m) > maxKeys { shrink(m) } } + add(KindName, ev.Name) + add(KindClient, client) + switch { + case blocked: + // Counted by handleRewrite; not a name that doesn't exist. + case ev.RCode == "NXDOMAIN": + add(KindNXDomain, ev.Name) + case ev.RCode == "SERVFAIL": + add(KindServFail, ev.Name) + case ev.RCode == "REFUSED": + add(KindRefused, client) + } + for _, k := range Kinds { + if m := hours[hour][k]; len(m) > maxKeys { + shrink(m) + } + } +} + +// countView counts a per-view copy of a counter. +func (c *Collector) countView(minute int64, view, key string, n int64) { + if !c.viewsSince { + c.viewsSince = true + c.viewsSinceAt = minute + } + c.count(minute, ViewKey(view, key), n) } // handleRewrite counts a query that blocking answered: "blocked", and @@ -412,15 +449,24 @@ func (c *Collector) handleRewrite(rw Rewrite) { } c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time) + view := rw.View + if view == "" { + view = "_default" + } c.count(minute, "blocked", 1) c.count(minute, "blocklist:"+policySource(rw.Via), 1) + c.countView(minute, view, "blocked", 1) + c.countView(minute, view, "blocklist:"+policySource(rw.Via), 1) hours := c.hours if hour < c.liveHour { hours = c.late } hours[hour] = c.hourMaps(hours, hour) + client := clientKey(rw.Client, c.opts.Anonymize) hours[hour][KindBlocked][rw.Name]++ - hours[hour][KindBlockedClient][clientKey(rw.Client, c.opts.Anonymize)]++ + hours[hour][KindBlockedClient][client]++ + c.topsMap(hours, hour, ViewKind(KindBlocked, view))[rw.Name]++ + c.topsMap(hours, hour, ViewKind(KindBlockedClient, view))[client]++ } // handleRateLimit counts a response rate limiting limited, or would have: @@ -437,7 +483,12 @@ func (c *Collector) handleRateLimit(rl RateLimit) { key = "rrl:would-" + rl.Action } minute, hour := c.minuteOf(rl.Time), c.hourOf(rl.Time) + view := rl.View + if view == "" { + view = "_default" + } c.count(minute, key, 1) + c.countView(minute, view, key, 1) hours := c.hours if hour < c.liveHour { hours = c.late @@ -445,6 +496,8 @@ func (c *Collector) handleRateLimit(rl RateLimit) { hours[hour] = c.hourMaps(hours, hour) hours[hour][KindRateLimited][rl.Prefix]++ hours[hour][KindRateLimitedName][rl.Name+" "+rl.Type]++ + c.topsMap(hours, hour, ViewKind(KindRateLimited, view))[rl.Prefix]++ + c.topsMap(hours, hour, ViewKind(KindRateLimitedName, view))[rl.Name+" "+rl.Type]++ } // policySource names what a rewrite came from: a blocklist's name, "rules" @@ -547,6 +600,10 @@ func (c *Collector) flush(ctx context.Context, final bool) { b.AddCounters(ResHour, c.hourOf(t), vals) b.AddCounters(ResDay, c.dayOf(t), vals) delete(c.minutes, m) + if c.viewsSinceAt != 0 { + b.SetMeta("views_since", strconv.FormatInt(c.viewsSinceAt, 10)) + c.viewsSinceAt = 0 + } if first && len(vals) > 0 { ft := t c.status.FirstData = &ft diff --git a/internal/stats/query.go b/internal/stats/query.go index dba9b97..7c68be3 100644 --- a/internal/stats/query.go +++ b/internal/stats/query.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "slices" + "strconv" "strings" "time" ) @@ -14,6 +15,110 @@ type Querier struct { Store *Store Opts Options Now func() time.Time + // Views limits the figures to these views (none: every view). Only + // what named's response log says can be split by view; named's own + // counters (cache, transport, round trips, ...) can't. + Views []string +} + +// ForViews returns a querier limited to views (all views when empty). +func (q *Querier) ForViews(views []string) *Querier { + c := *q + c.Views = views + return &c +} + +func (q *Querier) filtered() bool { return len(q.Views) > 0 } + +// perView reports whether a counter key exists per view, and its name +// there ("queries" is counted as responses). +func perView(key string) (string, bool) { + if key == "queries" || key == "responses" { + return "responses", true + } + if key == "blocked" { + return key, true + } + for _, p := range []string{"rcode:", "qtype:", "zone:", "blocklist:", "rrl:"} { + if strings.HasPrefix(key, p) || key+":" == p { + return key, true + } + } + return key, false +} + +// PerViewGroups are the Breakdown groups that exist per view. +var PerViewGroups = []string{"qtype", "rcode", "view", "zone", "blocklist", "rrl"} + +// ErrNotPerView is returned for figures named only keeps for the whole server. +var ErrNotPerView = fmt.Errorf("named keeps this for the whole server only, not per view") + +// sums is Store.sums, limited to the querier's views. +func (q *Querier) sums(ctx context.Context, res string, from, to int64, prefix string) (map[string]int64, error) { + if !q.filtered() { + return q.Store.sums(ctx, res, from, to, prefix) + } + if prefix == "view:" { + all, err := q.Store.sums(ctx, res, from, to, prefix) + if err != nil { + return nil, err + } + out := map[string]int64{} + for _, v := range q.Views { + if n := all["view:"+v]; n != 0 { + out["view:"+v] = n + } + } + return out, nil + } + if _, ok := perView(prefix); !ok { + // Not kept per view (named's own counters): the whole server's. + return q.Store.sums(ctx, res, from, to, prefix) + } + out := map[string]int64{} + for _, v := range q.Views { + m, err := q.Store.sums(ctx, res, from, to, ViewKey(v, prefix)) + if err != nil { + return nil, err + } + for k, n := range m { + out[strings.TrimPrefix(k, ViewKey(v, ""))] += n + } + } + return out, nil +} + +// series is Store.series, limited to the querier's views. +func (q *Querier) series(ctx context.Context, res string, from, to int64, key string) (map[int64]int64, error) { + pv, ok := perView(key) + if !q.filtered() || !ok { + return q.Store.series(ctx, res, from, to, key) + } + out := map[int64]int64{} + for _, v := range q.Views { + m, err := q.Store.series(ctx, res, from, to, ViewKey(v, pv)) + if err != nil { + return nil, err + } + for ts, n := range m { + out[ts] += n + } + } + return out, nil +} + +// ViewsSince is when per-view figures start (nil before any). +func (q *Querier) ViewsSince(ctx context.Context) *time.Time { + v, err := q.Store.Meta(ctx, "views_since") + if err != nil || v == "" { + return nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil + } + t := time.Unix(n, 0) + return &t } // NewQuerier returns a querier for a collector's data. @@ -105,10 +210,15 @@ func (s *Store) series(ctx context.Context, res string, from, to int64, key stri return out, rows.Err() } -// tops sums top lists in [from, to). -func (s *Store) tops(ctx context.Context, res, kind string, from, to int64, limit int) ([]Entry, error) { - rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(count) AS n FROM tops WHERE res = ? AND kind = ? AND ts >= ? AND ts < ? - GROUP BY key ORDER BY n DESC, key LIMIT ?`, res, kind, from, to, limit) +// tops sums top lists (of one or more kinds) in [from, to). +func (s *Store) tops(ctx context.Context, res string, kinds []string, from, to int64, limit int) ([]Entry, error) { + args := []any{res} + for _, k := range kinds { + args = append(args, k) + } + args = append(args, from, to, limit) + rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(count) AS n FROM tops WHERE res = ? AND kind IN (`+strings.TrimSuffix(strings.Repeat("?,", len(kinds)), ",")+`) AND ts >= ? AND ts < ? + GROUP BY key ORDER BY n DESC, key LIMIT ?`, args...) if err != nil { return nil, err } @@ -167,6 +277,11 @@ type Summary struct { BlockedRatio *float64 `json:"blocked_ratio,omitempty"` Blocklists []Entry `json:"blocklists,omitempty"` Tops map[string][]Entry `json:"tops,omitempty"` + // Views and ViewsSince are set when the figures are for some views + // only: queries then counts responses, and named's own counters + // (answers, transport, cache) are left out. + FilterViews []string `json:"filter_views,omitempty"` + ViewsSince *time.Time `json:"views_since,omitempty"` } // Summary summarizes the last since. @@ -179,9 +294,32 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int if err != nil { return nil, err } + if q.filtered() { + // The chosen views' counters, under their usual names. + views := map[string]int64{} + for _, v := range q.Views { + views["view:"+v] = all["view:"+v] + } + picked := map[string]int64{} + for k, n := range all { + for _, v := range q.Views { + if rest, ok := strings.CutPrefix(k, ViewKey(v, "")); ok { + picked[rest] += n + } + } + } + for k, n := range views { + picked[k] = n + } + picked["queries"] = picked["responses"] + all = picked + } s := &Summary{From: time.Unix(from, 0), To: now, Resolution: res, Queries: all["queries"], RCodes: sorted(all, "rcode:"), QTypes: sorted(all, "qtype:"), Views: sorted(all, "view:"), Zones: sorted(all, "zone:"), Answers: sorted(all, "answer:"), Transport: sorted(all, "transport:")} + if q.filtered() { + s.FilterViews, s.ViewsSince = q.Views, q.ViewsSince(ctx) + } if h, m := all["cache:hits"], all["cache:misses"]; h+m > 0 { r := float64(h) / float64(h+m) s.CacheHitRatio = &r @@ -195,7 +333,7 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int s.BlockedRatio = &b } } - per, err := q.Store.series(ctx, res, from, to, "queries") + per, err := q.series(ctx, res, from, to, "queries") if err != nil { return nil, err } @@ -237,7 +375,14 @@ func (q *Querier) Top(ctx context.Context, kind string, since time.Duration, lim } now := q.Now() res := q.topsResolution(since) - return q.Store.tops(ctx, res, kind, q.bucketStart(res, now.Add(-since)), now.Unix()+1, limit) + kinds := []string{kind} + if q.filtered() { + kinds = nil + for _, v := range q.Views { + kinds = append(kinds, ViewKind(kind, v)) + } + } + return q.Store.tops(ctx, res, kinds, q.bucketStart(res, now.Add(-since)), now.Unix()+1, limit) } // Breakdown returns the totals of a counter group (qtype, rcode, view, @@ -246,9 +391,12 @@ func (q *Querier) Breakdown(ctx context.Context, group string, since time.Durati if !slices.Contains(Groups, group) { return nil, fmt.Errorf("unknown group %q (use %s)", group, strings.Join(Groups, ", ")) } + if q.filtered() && !slices.Contains(PerViewGroups, group) { + return nil, ErrNotPerView + } now := q.Now() res := q.Resolution(since) - m, err := q.Store.sums(ctx, res, q.bucketStart(res, now.Add(-since)), now.Unix()+1, group+":") + m, err := q.sums(ctx, res, q.bucketStart(res, now.Add(-since)), now.Unix()+1, group+":") if err != nil { return nil, err } @@ -329,7 +477,7 @@ func (q *Querier) Timeline(ctx context.Context, since time.Duration, step string var expanded []string for _, k := range keys { if g, ok := strings.CutSuffix(k, ":*"); ok { - m, err := q.Store.sums(ctx, st.res, from, to, g+":") + m, err := q.sums(ctx, st.res, from, to, g+":") if err != nil { return nil, err } @@ -342,7 +490,7 @@ func (q *Querier) Timeline(ctx context.Context, since time.Duration, step string } tl := &Timeline{Step: st.name, Times: starts, Series: []*Series{}} bucketed := func(key string) ([]int64, error) { - raw, err := q.Store.series(ctx, st.res, from, to, key) + raw, err := q.series(ctx, st.res, from, to, key) if err != nil { return nil, err } @@ -404,7 +552,7 @@ func (q *Querier) Heatmap(ctx context.Context, key string, since time.Duration) since = min(since, q.Opts.RetainHour) now := q.Now() from := q.bucketStart(ResHour, now.Add(-since)) - raw, err := q.Store.series(ctx, ResHour, from, now.Unix()+1, key) + raw, err := q.series(ctx, ResHour, from, now.Unix()+1, key) if err != nil { return nil, err } diff --git a/internal/stats/store.go b/internal/stats/store.go index faf4f9f..96227aa 100644 --- a/internal/stats/store.go +++ b/internal/stats/store.go @@ -264,3 +264,22 @@ func (s *Store) TopsAt(ctx context.Context, res string, ts int64, kind string) ( } return out, rows.Err() } + +// KindsAt returns the top-list kinds stored for one bucket (per-view +// kinds included). +func (s *Store) KindsAt(ctx context.Context, res string, ts int64) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT kind FROM tops WHERE res = ? AND ts = ?`, res, ts) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + return nil, err + } + out = append(out, k) + } + return out, rows.Err() +} diff --git a/internal/stats/views_test.go b/internal/stats/views_test.go new file mode 100644 index 0000000..e75601a --- /dev/null +++ b/internal/stats/views_test.go @@ -0,0 +1,103 @@ +package stats + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + "time" +) + +// Figures for chosen views: from the response log, per view, and added +// up for several. +func TestViewFilter(t *testing.T) { + ctx := context.Background() + loc := time.FixedZone("test", 0) + dir := t.TempDir() + logFile := filepath.Join(dir, "responses.log") + store, err := OpenStore(filepath.Join(dir, "stats.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + named := newFakeNamed(t) + clk := &clock{time.Date(2026, 10, 5, 10, 0, 30, 0, loc)} + opts := Options{QueryLog: true, LogFile: logFile, StatisticsChannel: "unused", TopN: 10, + RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour, Location: loc} + zones := func() map[string][]string { + return map[string][]string{"internal": {"adbraun.lol"}, "external": {"adbraun.lol"}} + } + c := NewCollector(store, opts, zones, slog.New(slog.NewTextHandler(io.Discard, nil))) + c.poller.URL = named.srv.URL + c.now = clk.now + c.start(ctx) + at := time.Date(2026, 10, 5, 10, 0, 40, 0, loc) + lines := []string{ + logLine(at, "10.10.0.37", "internal", "nas.adbraun.lol", "A", "NOERROR"), + logLine(at, "10.10.0.37", "internal", "nas.adbraun.lol", "AAAA", "NOERROR"), + logLine(at, "10.10.0.42", "internal", "nope.adbraun.lol", "A", "NXDOMAIN"), + logLine(at, "203.0.113.9", "external", "www.adbraun.lol", "A", "NOERROR"), + logLine(at, "203.0.113.9", "external", "x.example", "A", "REFUSED"), + logLine(at, "192.0.2.5", "vps", "debian.org", "AAAA", "NOERROR"), + } + f, _ := os.Create(logFile) + for _, l := range lines { + f.WriteString(l) + } + f.Close() + c.readLog() + named.add(6, 1) + clk.t = time.Date(2026, 10, 5, 10, 1, 2, 0, loc) + c.flush(ctx, false) + clk.t = time.Date(2026, 10, 5, 11, 0, 2, 0, loc) + c.flush(ctx, false) + + all := &Querier{Store: store, Opts: opts, Now: clk.now} + s, err := all.Summary(ctx, 24*time.Hour, 5) + if err != nil || s.Queries != 6 || s.FilterViews != nil { + t.Fatalf("all views: %+v %v", s, err) + } + in := all.ForViews([]string{"internal"}) + s, err = in.Summary(ctx, 24*time.Hour, 5) + if err != nil { + t.Fatal(err) + } + if s.Queries != 3 || len(s.Views) != 1 || s.Views[0].Key != "internal" || s.ViewsSince == nil { + t.Errorf("internal: queries %d views %v since %v", s.Queries, s.Views, s.ViewsSince) + } + if len(s.RCodes) != 2 || s.RCodes[0] != (Entry{Key: "NOERROR", Count: 2}) || len(s.QTypes) != 2 { + t.Errorf("internal rcodes %v qtypes %v", s.RCodes, s.QTypes) + } + if s.Tops[KindName][0] != (Entry{Key: "nas.adbraun.lol", Count: 2}) || len(s.Tops[KindNXDomain]) != 1 || len(s.Answers) != 0 { + t.Errorf("internal tops %v answers %v", s.Tops, s.Answers) + } + // Linked views, say: two together. + two := all.ForViews([]string{"internal", "external"}) + s, _ = two.Summary(ctx, 24*time.Hour, 5) + if s.Queries != 5 || len(s.Views) != 2 { + t.Errorf("internal+external: %d %v", s.Queries, s.Views) + } + if es, _ := two.Top(ctx, KindClient, 24*time.Hour, 5); len(es) != 3 || es[2].Key != "10.10.0.42" { + t.Errorf("clients of two views: %v", es) + } + if es, _ := two.Breakdown(ctx, "rcode", 24*time.Hour); len(es) != 3 { + t.Errorf("rcodes of two views: %v", es) + } + if _, err := two.Breakdown(ctx, "answer", 24*time.Hour); !errors.Is(err, ErrNotPerView) { + t.Errorf("answers per view: %v", err) + } + tl, err := all.ForViews([]string{"vps"}).Timeline(ctx, 2*time.Hour, "1m", []string{"queries", "rcode:*"}) + if err != nil { + t.Fatal(err) + } + var total int64 + for _, v := range tl.Series[0].Values { + total += v + } + if total != 1 || len(tl.Series) != 2 || tl.Series[1].Key != "rcode:NOERROR" { + t.Errorf("vps timeline: %d %+v", total, tl.Series) + } +} diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js index c4ed52c..634e29f 100644 --- a/internal/web/static/assets/pages/stats.js +++ b/internal/web/static/assets/pages/stats.js @@ -1,5 +1,6 @@ import { h, mount, fmtTime, relTime, errorBox } from "../ui.js"; import { get } from "../api.js"; +import { rerender } from "../app.js"; import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js"; const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"]]; @@ -17,6 +18,20 @@ const rttOrder = ["<10ms", "10-100ms", "100-500ms", "500-800ms", "800-1600ms", " let generation = 0; +// The views shown are remembered per browser: the ones switched off, so a +// new view shows up switched on. +const offKey = "jarl.stats.views.off"; +function viewsOff() { + try { return new Set(JSON.parse(localStorage.getItem(offKey) || "[]")); } catch { return new Set(); } +} +function saveViewsOff(set) { + try { localStorage.setItem(offKey, JSON.stringify([...set])); } catch { /* private window: not remembered */ } +} + +// Breakdowns that exist per view; the others are named's own counters, +// for the whole server. +const perViewGroups = ["qtype", "rcode", "view", "zone", "blocklist"]; + export async function render(el, params, query) { const gen = ++generation; const want = query && query.get ? query.get("since") : null; @@ -33,11 +48,21 @@ export async function render(el, params, query) { return; } const heatSince = rangeSecs[since] < 7 * 86400 ? "7d" : since; + // Which views: all of them unless some are switched off. + const allViews = status.query_log ? (await get("/views").catch(() => [])).map((v) => v.name) : []; + const off = viewsOff(); + const shown = allViews.filter((v) => !off.has(v)); + const filtered = allViews.length > 0 && shown.length < allViews.length; + const vf = filtered ? { views: shown.join(",") } : {}; + if (filtered && !shown.length) { + mount(el, head, filterRow(since, allViews, off), h("div", { class: "empty" }, "No views chosen: switch one on above.")); + return; + } let data; try { - data = await loadAll(since, heatSince, status.query_log); + data = await loadAll(since, heatSince, status.query_log, vf, filtered); } catch (e) { - mount(el, head, errorBox(e)); + mount(el, head, filterRow(since, allViews, off), errorBox(e)); return; } if (gen !== generation) return; // the user moved on @@ -46,13 +71,17 @@ export async function render(el, params, query) { // sudden drop in traffic. const timeline = complete(data.timeline), errors = complete(data.errors); const step = stepSeconds[timeline.step]; - const parts = [head, filterRow(since)]; + const parts = [head, filterRow(since, allViews, off)]; for (const w of status.warnings || []) parts.push(h("div", { class: "msg warn" }, w)); + if (filtered) { + parts.push(h("div", { class: "msg info small" }, "Showing view" + (shown.length > 1 ? "s " : " ") + shown.join(", ") + ": answers from named's response log" + + (s.views_since ? ", per view since " + fmtTime(s.views_since) : "") + ". named's own figures (answer sources, upstream times, cache, the server) are for the whole server.")); + } // Headline numbers. const rangeLabel = "in the last " + ranges.find((r) => r[0] === since)[1]; parts.push(h("div", { class: "cards" }, - tile("Queries", compact(s.queries), rangeLabel), + tile(filtered ? "Answers" : "Queries", compact(s.queries), rangeLabel), tile("Per second", s.avg_qps < 10 ? s.avg_qps.toFixed(2) : Math.round(s.avg_qps).toLocaleString(), "on average"), s.peak ? tile("Busiest " + s.peak.per, compact(s.peak.queries), fmtTime(s.peak.time)) : null, s.nxdomain_ratio != null ? tile("NXDOMAIN", pct(s.nxdomain_ratio), "of queries named no existing name") : null, @@ -62,7 +91,7 @@ export async function render(el, params, query) { // Volume over time: one series, so no legend box; the title names it. // With blocking, a second line shows how many of them were blocked. const q = timeline.series.find((x) => x.key === "queries") || { values: timeline.times.map(() => 0) }; - const qSeries = [{ label: "Queries", values: q.values, slot: 1 }]; + const qSeries = [{ label: filtered ? "Answers" : "Queries", values: q.values, slot: 1 }]; const bl = timeline.series.find((x) => x.key === "blocked"); if (bl && bl.values.some((v) => v > 0)) qSeries.push({ label: "Blocked", values: bl.values, slot: 2 }); // Failed answers: three fixed series, legend plus end labels. @@ -71,7 +100,8 @@ export async function render(el, params, query) { // The two charts are one above the other: the same room for labels on // both sides keeps their time axes in line. const gutter = { left: 52, right: Math.max(endLabelWidth(qSeries), anyFailures ? endLabelWidth(fSeries) : 0) }; - parts.push(chartCard(qSeries.length > 1 ? "Queries and blocked queries" : "Queries", "per " + stepName(timeline.step) + ", " + rangeLabel, + const qWord = filtered ? "Answers" : "Queries"; + parts.push(chartCard(qSeries.length > 1 ? qWord + " and blocked queries" : qWord, "per " + stepName(timeline.step) + ", " + rangeLabel, lineChart({ series: qSeries, times: timeline.times, step, area: true, gutter }), timeTable(qSeries, timeline.times, step), { wide: true, legend: qSeries.length > 1 ? legend(qSeries) : null })); @@ -117,19 +147,20 @@ export async function render(el, params, query) { scheduleRefresh(el, params, query, gen); } -async function loadAll(since, heatSince, queryLog) { +async function loadAll(since, heatSince, queryLog, vf, filtered) { const p = { - summary: get("/stats/summary", { since, top: 0 }), - timeline: get("/stats/timeline", { since, key: "queries,blocked" }), - errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(",") }), - heat: get("/stats/heatmap", { since: heatSince }), + summary: get("/stats/summary", { since, top: 0, ...vf }), + timeline: get("/stats/timeline", { since, key: "queries,blocked", ...vf }), + errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(","), ...vf }), + heat: get("/stats/heatmap", { since: heatSince, ...vf }), server: get("/stats/server").catch(() => null), gauges: get("/stats/timeline", { since, key: gaugeKeys.join(",") }), }; const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt", "blocklist"]; - groups.forEach((g) => { p["b_" + g] = get("/stats/breakdown", { group: g, since }); }); + // With some views only, named's own counters aren't asked for. + groups.forEach((g) => { p["b_" + g] = filtered && !perViewGroups.includes(g) ? Promise.resolve([]) : get("/stats/breakdown", { group: g, since, ...vf }); }); if (queryLog) { - for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked", "blocked_client"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 }); + for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked", "blocked_client"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15, ...vf }); } const keys = Object.keys(p); const vals = await Promise.all(Object.values(p)); @@ -162,11 +193,26 @@ function complete(tl) { return tl; } -function filterRow(since) { - return h("div", { class: "filter-row", role: "group", "aria-label": "Time range" }, +function filterRow(since, views, off) { + const range = h("div", { class: "filter-row", role: "group", "aria-label": "Time range" }, h("span", { class: "muted" }, "Range"), h("div", { class: "segmented" }, ranges.map(([k, label]) => h("a", { href: "#/stats?since=" + k, class: k === since ? "active" : "", "aria-current": k === since ? "true" : null, title: "last " + label }, k)))); + if (!views || views.length < 2) return range; + // A toggle per view (together: linked views, say), and all or none. + const set = (next) => { saveViewsOff(next); rerender(); }; + const toggles = views.map((v) => { + const on = !off.has(v); + return h("button", { type: "button", class: "chip toggle-chip" + (on ? " on" : ""), "aria-pressed": on ? "true" : "false", + title: on ? "Leave view " + v + " out" : "Include view " + v, + onclick: () => { const n = new Set(off); if (on) n.add(v); else n.delete(v); set(n); } }, v); + }); + return h("div", { class: "filter-rows" }, range, + h("div", { class: "filter-row", role: "group", "aria-label": "Views" }, + h("span", { class: "muted" }, "Views"), + h("div", { class: "chips" }, toggles), + h("button", { type: "button", class: "small", disabled: off.size === 0, onclick: () => set(new Set()) }, "All"), + h("button", { type: "button", class: "small", disabled: off.size === views.length, onclick: () => set(new Set(views)) }, "None"))); } const gaugeKeys = ["g:named:rss", "g:named:mem", "g:cache:mem", "g:named:cpu"]; diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index abe4653..41f92b3 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -522,3 +522,10 @@ table.pick-list tr.pick.selected { background: var(--accent-soft); } /* Encrypted DNS: a protocol per line with its port. */ .enc-ports > div { margin: 2px 0; } .enc-ports input[type=number] { display: inline-block; } + +/* Statistics: views switched on and off. */ +.filter-rows { display: flex; flex-direction: column; gap: 8px; margin-bottom: 12px; } +.filter-rows .filter-row { margin-bottom: 0; } +.filter-row .chips { display: flex; flex-wrap: wrap; gap: 6px; } +.chip.toggle-chip { opacity: 0.55; font-size: 13px; padding: 3px 12px; } +.chip.toggle-chip.on { opacity: 1; text-decoration: none; background: var(--accent-soft); border-color: var(--accent); }