diff --git a/CHANGELOG.md b/CHANGELOG.md index 0175694..beaf44c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,13 @@ All notable changes to Jarl are documented here. The format follows ### Added +- Statistics per view ([docs/stats.md](docs/stats.md#views)): the + Statistics page has a toggle per view, with All and None, remembered in + the browser, to see some views only or several together (linked views, + say). Answers, result codes, record types, zones, blocking, rate + limiting and the top lists follow it; named's own counters stay for the + whole server, and the page says so. `jarl stats --view`, API + `?views=`. Per-view figures start with this version. - DNS over TLS and HTTPS ([docs/encrypted-dns.md](docs/encrypted-dns.md)): `jarl encrypted-dns setup --hostname NAME --address ADDR` (and the Encrypted DNS page) obtains a certificate for the host name if needed, diff --git a/cmd/jarl/stats.go b/cmd/jarl/stats.go index 3f2a5de..226d180 100644 --- a/cmd/jarl/stats.go +++ b/cmd/jarl/stats.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "math" + "net/url" "slices" "strconv" "strings" @@ -21,8 +22,22 @@ func statsCmd() *cobra.Command { Long: `Query statistics collected from named (see docs/stats.md). They must be enabled in /etc/jarl/jarl.yaml (stats.enabled: true). -Ranges are given with --since: 90m, 24h, 7d, 2w. Use -o json or -o yaml to -export the data.`, +Ranges are given with --since: 90m, 24h, 7d, 2w. --view limits the figures +to some views (repeat it, or separate with commas): answers from named's +response log, per view; named's own counters stay for the whole server. +Use -o json or -o yaml to export the data.`, + } + var statsViews []string + cmd.PersistentFlags().StringSliceVar(&statsViews, "view", nil, "only these views (repeat, or comma separated)") + // withViews adds the --view filter to a query. + withViews := func(v url.Values) url.Values { + if len(statsViews) > 0 { + if v == nil { + v = url.Values{} + } + v.Set("views", strings.Join(statsViews, ",")) + } + return v } // Each command has its own --since variable: their defaults differ, and // a shared variable would take the last default for all of them. @@ -77,12 +92,21 @@ export the data.`, return err } var s stats.Summary - if err := c.Get(ctx(), "/stats/summary", q("since", summarySince, "top", strconv.Itoa(topN)), &s); err != nil { + if err := c.Get(ctx(), "/stats/summary", withViews(q("since", summarySince, "top", strconv.Itoa(topN))), &s); err != nil { return err } return output(s, func(w io.Writer) { fmt.Fprintf(w, "Range:\t%s to %s\n", s.From.Local().Format("2006-01-02 15:04"), s.To.Local().Format("2006-01-02 15:04")) - fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + if len(s.FilterViews) > 0 { + since := "" + if s.ViewsSince != nil { + since = ", per view since " + s.ViewsSince.Local().Format("2006-01-02 15:04") + } + fmt.Fprintf(w, "Views:\t%s (from the response log%s; named's own counters are left out)\n", strings.Join(s.FilterViews, ", "), since) + fmt.Fprintf(w, "Answers:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + } else { + fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) + } if s.Peak != nil { fmt.Fprintf(w, "Busiest %s:\t%s with %d queries\n", s.Peak.Per, s.Peak.Time.Local().Format("2006-01-02 15:04"), s.Peak.Queries) } @@ -157,7 +181,7 @@ export the data.`, return err } var es []stats.Entry - if err := c.Get(ctx(), "/stats/top", q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit)), &es); err != nil { + if err := c.Get(ctx(), "/stats/top", withViews(q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit))), &es); err != nil { return err } return output(es, func(w io.Writer) { @@ -207,7 +231,7 @@ export the data.`, qv.Add("key", k) } var tl stats.Timeline - if err := c.Get(ctx(), "/stats/timeline", qv, &tl); err != nil { + if err := c.Get(ctx(), "/stats/timeline", withViews(qv), &tl); err != nil { return err } return output(tl, func(w io.Writer) { @@ -256,7 +280,7 @@ export the data.`, return err } var hm stats.Heatmap - if err := c.Get(ctx(), "/stats/heatmap", q("since", heatmapSince, "key", hmKey), &hm); err != nil { + if err := c.Get(ctx(), "/stats/heatmap", withViews(q("since", heatmapSince, "key", hmKey)), &hm); err != nil { return err } return output(hm, func(w io.Writer) { @@ -308,7 +332,7 @@ export the data.`, return err } var es []stats.Entry - if err := c.Get(ctx(), "/stats/breakdown", q("group", args[0], "since", breakdownSince), &es); err != nil { + if err := c.Get(ctx(), "/stats/breakdown", withViews(q("group", args[0], "since", breakdownSince)), &es); err != nil { return err } return output(es, func(w io.Writer) { diff --git a/docs/api.md b/docs/api.md index c2dc004..f048737 100644 --- a/docs/api.md +++ b/docs/api.md @@ -539,6 +539,10 @@ in it and its records. ### Query statistics +Every statistics endpoint below takes `views=internal,vps` to limit the +figures to those views ([stats.md](stats.md#views)); a breakdown that only +exists for the whole server then answers `400 not_per_view`. + Needs `stats:read`. When statistics are disabled, every endpoint but `status` answers 404 `stats_disabled`. See [stats.md](stats.md). diff --git a/docs/cli.md b/docs/cli.md index e4d15b0..59fc7b2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -528,6 +528,9 @@ working. ## Query statistics +`--view internal,vps` (or repeated) limits any `jarl stats` command to those +views ([stats.md](stats.md#views)). + ```sh jarl stats status jarl stats summary --since 7d diff --git a/docs/stats.md b/docs/stats.md index 9c6595e..234db3d 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -184,6 +184,26 @@ jarl stats summary --since 30d -o json # export (also -o yaml) `operator` roles include it, `dns-editor` doesn't. Statistics show client addresses and what they look up, so grant it with care. +## Views + +With views, the Statistics page has a toggle per view (and **All** and +**None**): switch some off to see only the others, for example your +internal and VPS views without the internet's noise on the external one, +or several linked views together. The choice is remembered in your +browser. The CLI takes `--view` on every `jarl stats` command (repeat it, +or separate views with commas), and the API `?views=internal,vps`. + +Only what named's response log says can be split by view: answers (shown +as *answers* instead of queries), result codes, record types, zones, +blocking, rate limiting and the top lists. named's own counters (answer +sources, transport, upstream response times, DNSSEC, the cache and the +server) are for the whole server, and the page says so; asking for them +per view (`jarl stats breakdown answer --view internal`) is refused. + +Per-view figures are kept from the version that added them (Jarl 0.4.1); +the page says since when. They take some more room in the statistics +database, roughly in proportion to the number of views. + ## How data is kept - Minute counters are kept for `retention.minute`, hourly counters and top diff --git a/internal/server/h_stats.go b/internal/server/h_stats.go index bf72f4d..96d3a4e 100644 --- a/internal/server/h_stats.go +++ b/internal/server/h_stats.go @@ -1,6 +1,7 @@ package server import ( + "errors" "net/http" "net/netip" "path/filepath" @@ -14,14 +15,24 @@ import ( "jarl/internal/stats" ) -// statsQuerier checks permission and that statistics are enabled. -func (s *Server) statsQuerier(p *auth.Principal) (*stats.Querier, error) { +// statsQuerier checks permission and that statistics are enabled. With +// ?views=internal,vps the figures are for those views only. +func (s *Server) statsQuerier(r *http.Request, p *auth.Principal) (*stats.Querier, error) { if err := require(p, auth.Action{Resource: auth.ResStats}); err != nil { return nil, err } if s.statsQ == nil { return nil, &httpError{http.StatusNotFound, "stats_disabled", "query statistics are disabled; set stats.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/stats.md)"} } + var views []string + for _, v := range strings.Split(r.URL.Query().Get("views"), ",") { + if v = strings.TrimSpace(v); v != "" { + views = append(views, v) + } + } + if len(views) > 0 { + return s.statsQ.ForViews(views), nil + } return s.statsQ, nil } @@ -46,7 +57,7 @@ func (s *Server) statsStatus(w http.ResponseWriter, r *http.Request, p *auth.Pri } func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -66,7 +77,7 @@ func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Pr } func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -93,7 +104,7 @@ func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Princi } func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -124,7 +135,7 @@ func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.P } func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -143,7 +154,7 @@ func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Pr } func (s *Server) statsBreakdown(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - q, err := s.statsQuerier(p) + q, err := s.statsQuerier(r, p) if err != nil { return err } @@ -152,6 +163,9 @@ func (s *Server) statsBreakdown(w http.ResponseWriter, r *http.Request, p *auth. return err } es, err := q.Breakdown(r.Context(), r.URL.Query().Get("group"), since) + if errors.Is(err, stats.ErrNotPerView) { + return &httpError{http.StatusBadRequest, "not_per_view", r.URL.Query().Get("group") + ": " + err.Error()} + } if err != nil { return badRequest("%v", err) } @@ -247,7 +261,7 @@ func clientNames(st *model.State, want map[netip.Addr]bool, readable func(*model // limits (rndc status), and the space left on the file systems Jarl and // named write to. func (s *Server) statsServer(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { - if _, err := s.statsQuerier(p); err != nil { + if _, err := s.statsQuerier(r, p); err != nil { return err } out := map[string]any{"server": s.cfg.Stats.Server()} diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go index 685ab9a..57450cc 100644 --- a/internal/server/stats_test.go +++ b/internal/server/stats_test.go @@ -35,7 +35,10 @@ func TestStatsAPI(t *testing.T) { minute := now.Unix() - now.Unix()%60 - 120 hour := minute - minute%3600 b := &stats.Batch{} - vals := map[string]int64{"queries": 50, "rcode:NOERROR": 45, "rcode:NXDOMAIN": 5, "qtype:A": 30, "qtype:AAAA": 20, "zone:example.com": 40, "zone:-": 10} + vals := map[string]int64{"queries": 50, "rcode:NOERROR": 45, "rcode:NXDOMAIN": 5, "qtype:A": 30, "qtype:AAAA": 20, "zone:example.com": 40, "zone:-": 10, + "view:internal": 30, "view:external": 20, stats.ViewKey("internal", "responses"): 30, stats.ViewKey("external", "responses"): 20, + stats.ViewKey("internal", "rcode:NOERROR"): 30, stats.ViewKey("external", "rcode:NOERROR"): 15, stats.ViewKey("external", "rcode:NXDOMAIN"): 5, + "answer:authoritative": 40} b.AddCounters(stats.ResMinute, minute, vals) b.AddCounters(stats.ResHour, hour, vals) b.SetTops(stats.ResHour, hour, stats.KindName, []stats.Entry{{Key: "www.example.com", Count: 30}, {Key: "mail.example.com", Count: 10}}) @@ -47,6 +50,16 @@ func TestStatsAPI(t *testing.T) { c := stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) e := newAPIEnv(t, c) + // Some views only. + if sum := e.do("GET", "/stats/summary?since=1h&views=external", e.token, nil, 200); sum["queries"].(float64) != 20 || len(sum["rcodes"].([]any)) != 2 || sum["filter_views"] == nil { + t.Errorf("external summary = %v", sum) + } + if b := e.list("/stats/breakdown?group=rcode&since=1h&views=internal,external", e.token); len(b) != 2 || b[0]["count"].(float64) != 45 { + t.Errorf("rcodes of both views = %v", b) + } + if out, status := e.req("GET", "/stats/breakdown?group=answer&since=1h&views=internal", e.token, nil); status != 400 || !strings.Contains(string(out), "not_per_view") { + t.Errorf("answers per view = %d %s", status, out) + } if st := e.do("GET", "/stats/status", e.token, nil, 200); st["enabled"] != true { t.Errorf("status = %v", st) } diff --git a/internal/stats/collector.go b/internal/stats/collector.go index 775fd83..d052fb4 100644 --- a/internal/stats/collector.go +++ b/internal/stats/collector.go @@ -33,6 +33,13 @@ const ( KindRateLimitedName = "ratelimited_name" ) +// ViewKey is the key of a counter for one view: per-view copies of the +// counters that come from named's response log (docs/stats.md). +func ViewKey(view, key string) string { return "@" + view + "/" + key } + +// ViewKind is the kind of a top list for one view. +func ViewKind(kind, view string) string { return kind + "@" + view } + // Kinds lists the top-list kinds. var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused, KindBlocked, KindBlockedClient, KindRateLimited, KindRateLimitedName} @@ -115,6 +122,9 @@ type Collector struct { // rewritten holds queries named blocked (client#port|name) until their // response line arrives. rewritten map[string]bool + // viewsSince is set once per-view counters exist (meta views_since). + viewsSince bool + viewsSinceAt int64 // proc is the last sample of named's process, server the latest server // statistics (server.go). proc *namedProc @@ -212,6 +222,9 @@ func (c *Collector) start(ctx context.Context) { } } } + if v, _ := c.store.Meta(ctx, "views_since"); v != "" { + c.viewsSince = true + } if v, _ := c.store.Meta(ctx, "first_data"); v != "" { if n, err := strconv.ParseInt(v, 10, 64); err == nil { t := time.Unix(n, 0) @@ -227,7 +240,8 @@ func (c *Collector) start(ctx context.Context) { if last, err := strconv.ParseInt(merged, 10, 64); err == nil { from := max(last+3600, cur-48*3600) for h := from; h < cur; h += 3600 { - for _, k := range Kinds { + kinds, _ := c.store.KindsAt(ctx, ResHour, h) + for _, k := range kinds { if es, err := c.store.TopsAt(ctx, ResHour, h, k); err == nil && len(es) > 0 { b.AddTops(ResDay, c.dayOf(time.Unix(h, 0)), k, es, 3*c.opts.TopN) } @@ -235,7 +249,8 @@ func (c *Collector) start(ctx context.Context) { } } b.SetMeta("merged_through", strconv.FormatInt(cur-3600, 10)) - for _, k := range Kinds { + curKinds, _ := c.store.KindsAt(ctx, ResHour, cur) + for _, k := range curKinds { es, err := c.store.TopsAt(ctx, ResHour, cur, k) if err != nil || len(es) == 0 { continue @@ -294,7 +309,7 @@ func (c *Collector) count(minute int64, key string, n int64) { c.minutes[minute] = m } m[key] += n - if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") { + if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") && !strings.HasPrefix(key, "@") { c.totals[key] += n } } @@ -365,33 +380,55 @@ func (c *Collector) handleLine(line string) { if view == "" { view = "_default" } + zone := c.zoneOf(ev.Name, ev.View) c.count(minute, "responses", 1) c.count(minute, "view:"+view, 1) - c.count(minute, "zone:"+c.zoneOf(ev.Name, ev.View), 1) + c.count(minute, "zone:"+zone, 1) + // The same, per view, for the statistics of chosen views. + c.countView(minute, view, "responses", 1) + c.countView(minute, view, "zone:"+zone, 1) + c.countView(minute, view, "rcode:"+ev.RCode, 1) + c.countView(minute, view, "qtype:"+ev.Type, 1) client := clientKey(ev.Client, c.opts.Anonymize) hours := c.hours if hour < c.liveHour { hours = c.late } hours[hour] = c.hourMaps(hours, hour) - tops := hours[hour] - tops[KindName][ev.Name]++ - tops[KindClient][client]++ - switch { - case blocked: - // Counted by handleRewrite; not a name that doesn't exist. - case ev.RCode == "NXDOMAIN": - tops[KindNXDomain][ev.Name]++ - case ev.RCode == "SERVFAIL": - tops[KindServFail][ev.Name]++ - case ev.RCode == "REFUSED": - tops[KindRefused][client]++ - } - for _, m := range tops { + add := func(kind, key string) { + hours[hour][kind][key]++ + m := c.topsMap(hours, hour, ViewKind(kind, view)) + m[key]++ if len(m) > maxKeys { shrink(m) } } + add(KindName, ev.Name) + add(KindClient, client) + switch { + case blocked: + // Counted by handleRewrite; not a name that doesn't exist. + case ev.RCode == "NXDOMAIN": + add(KindNXDomain, ev.Name) + case ev.RCode == "SERVFAIL": + add(KindServFail, ev.Name) + case ev.RCode == "REFUSED": + add(KindRefused, client) + } + for _, k := range Kinds { + if m := hours[hour][k]; len(m) > maxKeys { + shrink(m) + } + } +} + +// countView counts a per-view copy of a counter. +func (c *Collector) countView(minute int64, view, key string, n int64) { + if !c.viewsSince { + c.viewsSince = true + c.viewsSinceAt = minute + } + c.count(minute, ViewKey(view, key), n) } // handleRewrite counts a query that blocking answered: "blocked", and @@ -412,15 +449,24 @@ func (c *Collector) handleRewrite(rw Rewrite) { } c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time) + view := rw.View + if view == "" { + view = "_default" + } c.count(minute, "blocked", 1) c.count(minute, "blocklist:"+policySource(rw.Via), 1) + c.countView(minute, view, "blocked", 1) + c.countView(minute, view, "blocklist:"+policySource(rw.Via), 1) hours := c.hours if hour < c.liveHour { hours = c.late } hours[hour] = c.hourMaps(hours, hour) + client := clientKey(rw.Client, c.opts.Anonymize) hours[hour][KindBlocked][rw.Name]++ - hours[hour][KindBlockedClient][clientKey(rw.Client, c.opts.Anonymize)]++ + hours[hour][KindBlockedClient][client]++ + c.topsMap(hours, hour, ViewKind(KindBlocked, view))[rw.Name]++ + c.topsMap(hours, hour, ViewKind(KindBlockedClient, view))[client]++ } // handleRateLimit counts a response rate limiting limited, or would have: @@ -437,7 +483,12 @@ func (c *Collector) handleRateLimit(rl RateLimit) { key = "rrl:would-" + rl.Action } minute, hour := c.minuteOf(rl.Time), c.hourOf(rl.Time) + view := rl.View + if view == "" { + view = "_default" + } c.count(minute, key, 1) + c.countView(minute, view, key, 1) hours := c.hours if hour < c.liveHour { hours = c.late @@ -445,6 +496,8 @@ func (c *Collector) handleRateLimit(rl RateLimit) { hours[hour] = c.hourMaps(hours, hour) hours[hour][KindRateLimited][rl.Prefix]++ hours[hour][KindRateLimitedName][rl.Name+" "+rl.Type]++ + c.topsMap(hours, hour, ViewKind(KindRateLimited, view))[rl.Prefix]++ + c.topsMap(hours, hour, ViewKind(KindRateLimitedName, view))[rl.Name+" "+rl.Type]++ } // policySource names what a rewrite came from: a blocklist's name, "rules" @@ -547,6 +600,10 @@ func (c *Collector) flush(ctx context.Context, final bool) { b.AddCounters(ResHour, c.hourOf(t), vals) b.AddCounters(ResDay, c.dayOf(t), vals) delete(c.minutes, m) + if c.viewsSinceAt != 0 { + b.SetMeta("views_since", strconv.FormatInt(c.viewsSinceAt, 10)) + c.viewsSinceAt = 0 + } if first && len(vals) > 0 { ft := t c.status.FirstData = &ft diff --git a/internal/stats/query.go b/internal/stats/query.go index dba9b97..7c68be3 100644 --- a/internal/stats/query.go +++ b/internal/stats/query.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "slices" + "strconv" "strings" "time" ) @@ -14,6 +15,110 @@ type Querier struct { Store *Store Opts Options Now func() time.Time + // Views limits the figures to these views (none: every view). Only + // what named's response log says can be split by view; named's own + // counters (cache, transport, round trips, ...) can't. + Views []string +} + +// ForViews returns a querier limited to views (all views when empty). +func (q *Querier) ForViews(views []string) *Querier { + c := *q + c.Views = views + return &c +} + +func (q *Querier) filtered() bool { return len(q.Views) > 0 } + +// perView reports whether a counter key exists per view, and its name +// there ("queries" is counted as responses). +func perView(key string) (string, bool) { + if key == "queries" || key == "responses" { + return "responses", true + } + if key == "blocked" { + return key, true + } + for _, p := range []string{"rcode:", "qtype:", "zone:", "blocklist:", "rrl:"} { + if strings.HasPrefix(key, p) || key+":" == p { + return key, true + } + } + return key, false +} + +// PerViewGroups are the Breakdown groups that exist per view. +var PerViewGroups = []string{"qtype", "rcode", "view", "zone", "blocklist", "rrl"} + +// ErrNotPerView is returned for figures named only keeps for the whole server. +var ErrNotPerView = fmt.Errorf("named keeps this for the whole server only, not per view") + +// sums is Store.sums, limited to the querier's views. +func (q *Querier) sums(ctx context.Context, res string, from, to int64, prefix string) (map[string]int64, error) { + if !q.filtered() { + return q.Store.sums(ctx, res, from, to, prefix) + } + if prefix == "view:" { + all, err := q.Store.sums(ctx, res, from, to, prefix) + if err != nil { + return nil, err + } + out := map[string]int64{} + for _, v := range q.Views { + if n := all["view:"+v]; n != 0 { + out["view:"+v] = n + } + } + return out, nil + } + if _, ok := perView(prefix); !ok { + // Not kept per view (named's own counters): the whole server's. + return q.Store.sums(ctx, res, from, to, prefix) + } + out := map[string]int64{} + for _, v := range q.Views { + m, err := q.Store.sums(ctx, res, from, to, ViewKey(v, prefix)) + if err != nil { + return nil, err + } + for k, n := range m { + out[strings.TrimPrefix(k, ViewKey(v, ""))] += n + } + } + return out, nil +} + +// series is Store.series, limited to the querier's views. +func (q *Querier) series(ctx context.Context, res string, from, to int64, key string) (map[int64]int64, error) { + pv, ok := perView(key) + if !q.filtered() || !ok { + return q.Store.series(ctx, res, from, to, key) + } + out := map[int64]int64{} + for _, v := range q.Views { + m, err := q.Store.series(ctx, res, from, to, ViewKey(v, pv)) + if err != nil { + return nil, err + } + for ts, n := range m { + out[ts] += n + } + } + return out, nil +} + +// ViewsSince is when per-view figures start (nil before any). +func (q *Querier) ViewsSince(ctx context.Context) *time.Time { + v, err := q.Store.Meta(ctx, "views_since") + if err != nil || v == "" { + return nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil + } + t := time.Unix(n, 0) + return &t } // NewQuerier returns a querier for a collector's data. @@ -105,10 +210,15 @@ func (s *Store) series(ctx context.Context, res string, from, to int64, key stri return out, rows.Err() } -// tops sums top lists in [from, to). -func (s *Store) tops(ctx context.Context, res, kind string, from, to int64, limit int) ([]Entry, error) { - rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(count) AS n FROM tops WHERE res = ? AND kind = ? AND ts >= ? AND ts < ? - GROUP BY key ORDER BY n DESC, key LIMIT ?`, res, kind, from, to, limit) +// tops sums top lists (of one or more kinds) in [from, to). +func (s *Store) tops(ctx context.Context, res string, kinds []string, from, to int64, limit int) ([]Entry, error) { + args := []any{res} + for _, k := range kinds { + args = append(args, k) + } + args = append(args, from, to, limit) + rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(count) AS n FROM tops WHERE res = ? AND kind IN (`+strings.TrimSuffix(strings.Repeat("?,", len(kinds)), ",")+`) AND ts >= ? AND ts < ? + GROUP BY key ORDER BY n DESC, key LIMIT ?`, args...) if err != nil { return nil, err } @@ -167,6 +277,11 @@ type Summary struct { BlockedRatio *float64 `json:"blocked_ratio,omitempty"` Blocklists []Entry `json:"blocklists,omitempty"` Tops map[string][]Entry `json:"tops,omitempty"` + // Views and ViewsSince are set when the figures are for some views + // only: queries then counts responses, and named's own counters + // (answers, transport, cache) are left out. + FilterViews []string `json:"filter_views,omitempty"` + ViewsSince *time.Time `json:"views_since,omitempty"` } // Summary summarizes the last since. @@ -179,9 +294,32 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int if err != nil { return nil, err } + if q.filtered() { + // The chosen views' counters, under their usual names. + views := map[string]int64{} + for _, v := range q.Views { + views["view:"+v] = all["view:"+v] + } + picked := map[string]int64{} + for k, n := range all { + for _, v := range q.Views { + if rest, ok := strings.CutPrefix(k, ViewKey(v, "")); ok { + picked[rest] += n + } + } + } + for k, n := range views { + picked[k] = n + } + picked["queries"] = picked["responses"] + all = picked + } s := &Summary{From: time.Unix(from, 0), To: now, Resolution: res, Queries: all["queries"], RCodes: sorted(all, "rcode:"), QTypes: sorted(all, "qtype:"), Views: sorted(all, "view:"), Zones: sorted(all, "zone:"), Answers: sorted(all, "answer:"), Transport: sorted(all, "transport:")} + if q.filtered() { + s.FilterViews, s.ViewsSince = q.Views, q.ViewsSince(ctx) + } if h, m := all["cache:hits"], all["cache:misses"]; h+m > 0 { r := float64(h) / float64(h+m) s.CacheHitRatio = &r @@ -195,7 +333,7 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int s.BlockedRatio = &b } } - per, err := q.Store.series(ctx, res, from, to, "queries") + per, err := q.series(ctx, res, from, to, "queries") if err != nil { return nil, err } @@ -237,7 +375,14 @@ func (q *Querier) Top(ctx context.Context, kind string, since time.Duration, lim } now := q.Now() res := q.topsResolution(since) - return q.Store.tops(ctx, res, kind, q.bucketStart(res, now.Add(-since)), now.Unix()+1, limit) + kinds := []string{kind} + if q.filtered() { + kinds = nil + for _, v := range q.Views { + kinds = append(kinds, ViewKind(kind, v)) + } + } + return q.Store.tops(ctx, res, kinds, q.bucketStart(res, now.Add(-since)), now.Unix()+1, limit) } // Breakdown returns the totals of a counter group (qtype, rcode, view, @@ -246,9 +391,12 @@ func (q *Querier) Breakdown(ctx context.Context, group string, since time.Durati if !slices.Contains(Groups, group) { return nil, fmt.Errorf("unknown group %q (use %s)", group, strings.Join(Groups, ", ")) } + if q.filtered() && !slices.Contains(PerViewGroups, group) { + return nil, ErrNotPerView + } now := q.Now() res := q.Resolution(since) - m, err := q.Store.sums(ctx, res, q.bucketStart(res, now.Add(-since)), now.Unix()+1, group+":") + m, err := q.sums(ctx, res, q.bucketStart(res, now.Add(-since)), now.Unix()+1, group+":") if err != nil { return nil, err } @@ -329,7 +477,7 @@ func (q *Querier) Timeline(ctx context.Context, since time.Duration, step string var expanded []string for _, k := range keys { if g, ok := strings.CutSuffix(k, ":*"); ok { - m, err := q.Store.sums(ctx, st.res, from, to, g+":") + m, err := q.sums(ctx, st.res, from, to, g+":") if err != nil { return nil, err } @@ -342,7 +490,7 @@ func (q *Querier) Timeline(ctx context.Context, since time.Duration, step string } tl := &Timeline{Step: st.name, Times: starts, Series: []*Series{}} bucketed := func(key string) ([]int64, error) { - raw, err := q.Store.series(ctx, st.res, from, to, key) + raw, err := q.series(ctx, st.res, from, to, key) if err != nil { return nil, err } @@ -404,7 +552,7 @@ func (q *Querier) Heatmap(ctx context.Context, key string, since time.Duration) since = min(since, q.Opts.RetainHour) now := q.Now() from := q.bucketStart(ResHour, now.Add(-since)) - raw, err := q.Store.series(ctx, ResHour, from, now.Unix()+1, key) + raw, err := q.series(ctx, ResHour, from, now.Unix()+1, key) if err != nil { return nil, err } diff --git a/internal/stats/store.go b/internal/stats/store.go index faf4f9f..96227aa 100644 --- a/internal/stats/store.go +++ b/internal/stats/store.go @@ -264,3 +264,22 @@ func (s *Store) TopsAt(ctx context.Context, res string, ts int64, kind string) ( } return out, rows.Err() } + +// KindsAt returns the top-list kinds stored for one bucket (per-view +// kinds included). +func (s *Store) KindsAt(ctx context.Context, res string, ts int64) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT kind FROM tops WHERE res = ? AND ts = ?`, res, ts) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + return nil, err + } + out = append(out, k) + } + return out, rows.Err() +} diff --git a/internal/stats/views_test.go b/internal/stats/views_test.go new file mode 100644 index 0000000..e75601a --- /dev/null +++ b/internal/stats/views_test.go @@ -0,0 +1,103 @@ +package stats + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + "time" +) + +// Figures for chosen views: from the response log, per view, and added +// up for several. +func TestViewFilter(t *testing.T) { + ctx := context.Background() + loc := time.FixedZone("test", 0) + dir := t.TempDir() + logFile := filepath.Join(dir, "responses.log") + store, err := OpenStore(filepath.Join(dir, "stats.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + named := newFakeNamed(t) + clk := &clock{time.Date(2026, 10, 5, 10, 0, 30, 0, loc)} + opts := Options{QueryLog: true, LogFile: logFile, StatisticsChannel: "unused", TopN: 10, + RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour, Location: loc} + zones := func() map[string][]string { + return map[string][]string{"internal": {"adbraun.lol"}, "external": {"adbraun.lol"}} + } + c := NewCollector(store, opts, zones, slog.New(slog.NewTextHandler(io.Discard, nil))) + c.poller.URL = named.srv.URL + c.now = clk.now + c.start(ctx) + at := time.Date(2026, 10, 5, 10, 0, 40, 0, loc) + lines := []string{ + logLine(at, "10.10.0.37", "internal", "nas.adbraun.lol", "A", "NOERROR"), + logLine(at, "10.10.0.37", "internal", "nas.adbraun.lol", "AAAA", "NOERROR"), + logLine(at, "10.10.0.42", "internal", "nope.adbraun.lol", "A", "NXDOMAIN"), + logLine(at, "203.0.113.9", "external", "www.adbraun.lol", "A", "NOERROR"), + logLine(at, "203.0.113.9", "external", "x.example", "A", "REFUSED"), + logLine(at, "192.0.2.5", "vps", "debian.org", "AAAA", "NOERROR"), + } + f, _ := os.Create(logFile) + for _, l := range lines { + f.WriteString(l) + } + f.Close() + c.readLog() + named.add(6, 1) + clk.t = time.Date(2026, 10, 5, 10, 1, 2, 0, loc) + c.flush(ctx, false) + clk.t = time.Date(2026, 10, 5, 11, 0, 2, 0, loc) + c.flush(ctx, false) + + all := &Querier{Store: store, Opts: opts, Now: clk.now} + s, err := all.Summary(ctx, 24*time.Hour, 5) + if err != nil || s.Queries != 6 || s.FilterViews != nil { + t.Fatalf("all views: %+v %v", s, err) + } + in := all.ForViews([]string{"internal"}) + s, err = in.Summary(ctx, 24*time.Hour, 5) + if err != nil { + t.Fatal(err) + } + if s.Queries != 3 || len(s.Views) != 1 || s.Views[0].Key != "internal" || s.ViewsSince == nil { + t.Errorf("internal: queries %d views %v since %v", s.Queries, s.Views, s.ViewsSince) + } + if len(s.RCodes) != 2 || s.RCodes[0] != (Entry{Key: "NOERROR", Count: 2}) || len(s.QTypes) != 2 { + t.Errorf("internal rcodes %v qtypes %v", s.RCodes, s.QTypes) + } + if s.Tops[KindName][0] != (Entry{Key: "nas.adbraun.lol", Count: 2}) || len(s.Tops[KindNXDomain]) != 1 || len(s.Answers) != 0 { + t.Errorf("internal tops %v answers %v", s.Tops, s.Answers) + } + // Linked views, say: two together. + two := all.ForViews([]string{"internal", "external"}) + s, _ = two.Summary(ctx, 24*time.Hour, 5) + if s.Queries != 5 || len(s.Views) != 2 { + t.Errorf("internal+external: %d %v", s.Queries, s.Views) + } + if es, _ := two.Top(ctx, KindClient, 24*time.Hour, 5); len(es) != 3 || es[2].Key != "10.10.0.42" { + t.Errorf("clients of two views: %v", es) + } + if es, _ := two.Breakdown(ctx, "rcode", 24*time.Hour); len(es) != 3 { + t.Errorf("rcodes of two views: %v", es) + } + if _, err := two.Breakdown(ctx, "answer", 24*time.Hour); !errors.Is(err, ErrNotPerView) { + t.Errorf("answers per view: %v", err) + } + tl, err := all.ForViews([]string{"vps"}).Timeline(ctx, 2*time.Hour, "1m", []string{"queries", "rcode:*"}) + if err != nil { + t.Fatal(err) + } + var total int64 + for _, v := range tl.Series[0].Values { + total += v + } + if total != 1 || len(tl.Series) != 2 || tl.Series[1].Key != "rcode:NOERROR" { + t.Errorf("vps timeline: %d %+v", total, tl.Series) + } +} diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js index c4ed52c..634e29f 100644 --- a/internal/web/static/assets/pages/stats.js +++ b/internal/web/static/assets/pages/stats.js @@ -1,5 +1,6 @@ import { h, mount, fmtTime, relTime, errorBox } from "../ui.js"; import { get } from "../api.js"; +import { rerender } from "../app.js"; import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js"; const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"]]; @@ -17,6 +18,20 @@ const rttOrder = ["<10ms", "10-100ms", "100-500ms", "500-800ms", "800-1600ms", " let generation = 0; +// The views shown are remembered per browser: the ones switched off, so a +// new view shows up switched on. +const offKey = "jarl.stats.views.off"; +function viewsOff() { + try { return new Set(JSON.parse(localStorage.getItem(offKey) || "[]")); } catch { return new Set(); } +} +function saveViewsOff(set) { + try { localStorage.setItem(offKey, JSON.stringify([...set])); } catch { /* private window: not remembered */ } +} + +// Breakdowns that exist per view; the others are named's own counters, +// for the whole server. +const perViewGroups = ["qtype", "rcode", "view", "zone", "blocklist"]; + export async function render(el, params, query) { const gen = ++generation; const want = query && query.get ? query.get("since") : null; @@ -33,11 +48,21 @@ export async function render(el, params, query) { return; } const heatSince = rangeSecs[since] < 7 * 86400 ? "7d" : since; + // Which views: all of them unless some are switched off. + const allViews = status.query_log ? (await get("/views").catch(() => [])).map((v) => v.name) : []; + const off = viewsOff(); + const shown = allViews.filter((v) => !off.has(v)); + const filtered = allViews.length > 0 && shown.length < allViews.length; + const vf = filtered ? { views: shown.join(",") } : {}; + if (filtered && !shown.length) { + mount(el, head, filterRow(since, allViews, off), h("div", { class: "empty" }, "No views chosen: switch one on above.")); + return; + } let data; try { - data = await loadAll(since, heatSince, status.query_log); + data = await loadAll(since, heatSince, status.query_log, vf, filtered); } catch (e) { - mount(el, head, errorBox(e)); + mount(el, head, filterRow(since, allViews, off), errorBox(e)); return; } if (gen !== generation) return; // the user moved on @@ -46,13 +71,17 @@ export async function render(el, params, query) { // sudden drop in traffic. const timeline = complete(data.timeline), errors = complete(data.errors); const step = stepSeconds[timeline.step]; - const parts = [head, filterRow(since)]; + const parts = [head, filterRow(since, allViews, off)]; for (const w of status.warnings || []) parts.push(h("div", { class: "msg warn" }, w)); + if (filtered) { + parts.push(h("div", { class: "msg info small" }, "Showing view" + (shown.length > 1 ? "s " : " ") + shown.join(", ") + ": answers from named's response log" + + (s.views_since ? ", per view since " + fmtTime(s.views_since) : "") + ". named's own figures (answer sources, upstream times, cache, the server) are for the whole server.")); + } // Headline numbers. const rangeLabel = "in the last " + ranges.find((r) => r[0] === since)[1]; parts.push(h("div", { class: "cards" }, - tile("Queries", compact(s.queries), rangeLabel), + tile(filtered ? "Answers" : "Queries", compact(s.queries), rangeLabel), tile("Per second", s.avg_qps < 10 ? s.avg_qps.toFixed(2) : Math.round(s.avg_qps).toLocaleString(), "on average"), s.peak ? tile("Busiest " + s.peak.per, compact(s.peak.queries), fmtTime(s.peak.time)) : null, s.nxdomain_ratio != null ? tile("NXDOMAIN", pct(s.nxdomain_ratio), "of queries named no existing name") : null, @@ -62,7 +91,7 @@ export async function render(el, params, query) { // Volume over time: one series, so no legend box; the title names it. // With blocking, a second line shows how many of them were blocked. const q = timeline.series.find((x) => x.key === "queries") || { values: timeline.times.map(() => 0) }; - const qSeries = [{ label: "Queries", values: q.values, slot: 1 }]; + const qSeries = [{ label: filtered ? "Answers" : "Queries", values: q.values, slot: 1 }]; const bl = timeline.series.find((x) => x.key === "blocked"); if (bl && bl.values.some((v) => v > 0)) qSeries.push({ label: "Blocked", values: bl.values, slot: 2 }); // Failed answers: three fixed series, legend plus end labels. @@ -71,7 +100,8 @@ export async function render(el, params, query) { // The two charts are one above the other: the same room for labels on // both sides keeps their time axes in line. const gutter = { left: 52, right: Math.max(endLabelWidth(qSeries), anyFailures ? endLabelWidth(fSeries) : 0) }; - parts.push(chartCard(qSeries.length > 1 ? "Queries and blocked queries" : "Queries", "per " + stepName(timeline.step) + ", " + rangeLabel, + const qWord = filtered ? "Answers" : "Queries"; + parts.push(chartCard(qSeries.length > 1 ? qWord + " and blocked queries" : qWord, "per " + stepName(timeline.step) + ", " + rangeLabel, lineChart({ series: qSeries, times: timeline.times, step, area: true, gutter }), timeTable(qSeries, timeline.times, step), { wide: true, legend: qSeries.length > 1 ? legend(qSeries) : null })); @@ -117,19 +147,20 @@ export async function render(el, params, query) { scheduleRefresh(el, params, query, gen); } -async function loadAll(since, heatSince, queryLog) { +async function loadAll(since, heatSince, queryLog, vf, filtered) { const p = { - summary: get("/stats/summary", { since, top: 0 }), - timeline: get("/stats/timeline", { since, key: "queries,blocked" }), - errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(",") }), - heat: get("/stats/heatmap", { since: heatSince }), + summary: get("/stats/summary", { since, top: 0, ...vf }), + timeline: get("/stats/timeline", { since, key: "queries,blocked", ...vf }), + errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(","), ...vf }), + heat: get("/stats/heatmap", { since: heatSince, ...vf }), server: get("/stats/server").catch(() => null), gauges: get("/stats/timeline", { since, key: gaugeKeys.join(",") }), }; const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt", "blocklist"]; - groups.forEach((g) => { p["b_" + g] = get("/stats/breakdown", { group: g, since }); }); + // With some views only, named's own counters aren't asked for. + groups.forEach((g) => { p["b_" + g] = filtered && !perViewGroups.includes(g) ? Promise.resolve([]) : get("/stats/breakdown", { group: g, since, ...vf }); }); if (queryLog) { - for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked", "blocked_client"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 }); + for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked", "blocked_client"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15, ...vf }); } const keys = Object.keys(p); const vals = await Promise.all(Object.values(p)); @@ -162,11 +193,26 @@ function complete(tl) { return tl; } -function filterRow(since) { - return h("div", { class: "filter-row", role: "group", "aria-label": "Time range" }, +function filterRow(since, views, off) { + const range = h("div", { class: "filter-row", role: "group", "aria-label": "Time range" }, h("span", { class: "muted" }, "Range"), h("div", { class: "segmented" }, ranges.map(([k, label]) => h("a", { href: "#/stats?since=" + k, class: k === since ? "active" : "", "aria-current": k === since ? "true" : null, title: "last " + label }, k)))); + if (!views || views.length < 2) return range; + // A toggle per view (together: linked views, say), and all or none. + const set = (next) => { saveViewsOff(next); rerender(); }; + const toggles = views.map((v) => { + const on = !off.has(v); + return h("button", { type: "button", class: "chip toggle-chip" + (on ? " on" : ""), "aria-pressed": on ? "true" : "false", + title: on ? "Leave view " + v + " out" : "Include view " + v, + onclick: () => { const n = new Set(off); if (on) n.add(v); else n.delete(v); set(n); } }, v); + }); + return h("div", { class: "filter-rows" }, range, + h("div", { class: "filter-row", role: "group", "aria-label": "Views" }, + h("span", { class: "muted" }, "Views"), + h("div", { class: "chips" }, toggles), + h("button", { type: "button", class: "small", disabled: off.size === 0, onclick: () => set(new Set()) }, "All"), + h("button", { type: "button", class: "small", disabled: off.size === views.length, onclick: () => set(new Set(views)) }, "None"))); } const gaugeKeys = ["g:named:rss", "g:named:mem", "g:cache:mem", "g:named:cpu"]; diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index abe4653..41f92b3 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -522,3 +522,10 @@ table.pick-list tr.pick.selected { background: var(--accent-soft); } /* Encrypted DNS: a protocol per line with its port. */ .enc-ports > div { margin: 2px 0; } .enc-ports input[type=number] { display: inline-block; } + +/* Statistics: views switched on and off. */ +.filter-rows { display: flex; flex-direction: column; gap: 8px; margin-bottom: 12px; } +.filter-rows .filter-row { margin-bottom: 0; } +.filter-row .chips { display: flex; flex-wrap: wrap; gap: 6px; } +.chip.toggle-chip { opacity: 0.55; font-size: 13px; padding: 3px 12px; } +.chip.toggle-chip.on { opacity: 1; text-decoration: none; background: var(--accent-soft); border-color: var(--accent); }