Files
jarl/cmd/jarl/stats.go
T
adamandClaude Opus 5.5 c7e5bbb628
release / release (push) Successful in 2m4s
Release 0.5.0-rc.1
0.4.1's release candidates grew into 0.5.0: exposure checks, certificates, linked zones, encrypted DNS and per-view statistics are new features. The upgrade from 0.4.0 packages was tested in a Debian 13 container. The CLI's filtered stats summary says answers for the busiest minute too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 20:19:38 +01:00

510 lines
17 KiB
Go

package main
import (
"fmt"
"io"
"math"
"net/url"
"slices"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
"jarl/internal/stats"
)
func statsCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "stats",
Short: "Query statistics: volume over time, top names and clients, result codes",
Long: `Query statistics collected from named (see docs/stats.md). They must be
enabled in /etc/jarl/jarl.yaml (stats.enabled: true).
Ranges are given with --since: 90m, 24h, 7d, 2w. --view limits the figures
to some views (repeat it, or separate with commas): answers from named's
response log, per view; named's own counters stay for the whole server.
Use -o json or -o yaml to export the data.`,
}
var statsViews []string
cmd.PersistentFlags().StringSliceVar(&statsViews, "view", nil, "only these views (repeat, or comma separated)")
// withViews adds the --view filter to a query.
withViews := func(v url.Values) url.Values {
if len(statsViews) > 0 {
if v == nil {
v = url.Values{}
}
v.Set("views", strings.Join(statsViews, ","))
}
return v
}
// Each command has its own --since variable: their defaults differ, and
// a shared variable would take the last default for all of them.
var summarySince, topSince, timelineSince, heatmapSince, breakdownSince string
sinceFlag := func(c *cobra.Command, v *string, def, help string) {
c.Flags().StringVar(v, "since", def, help)
}
const sinceHelp = "time range: 90m, 24h, 7d, 2w"
status := &cobra.Command{
Use: "status",
Short: "Show whether statistics are collected and any problems",
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
var st stats.Status
if err := c.Get(ctx(), "/stats/status", nil, &st); err != nil {
return err
}
return output(st, func(w io.Writer) {
if !st.Enabled {
fmt.Fprintln(w, "Statistics:\tdisabled (set stats.enabled: true in /etc/jarl/jarl.yaml and restart jarl)")
return
}
mode := "counters and response log"
if !st.QueryLog {
mode = "counters only"
}
fmt.Fprintf(w, "Statistics:\tenabled (%s)\n", mode)
fmt.Fprintf(w, "Statistics channel:\t%s, last read %s\n", st.StatisticsChannel, fmtTime(st.LastPoll))
if st.QueryLog {
fmt.Fprintf(w, "Response log:\t%s, %d lines read, last %s\n", st.LogFile, st.LinesRead, fmtTime(st.LastLine))
}
fmt.Fprintf(w, "Data since:\t%s\n", fmtTime(st.FirstData))
fmt.Fprintf(w, "Retention:\tminutes %s, hours %s, days %s\n", st.Retention.Minute, st.Retention.Hour, st.Retention.Day)
for _, warn := range st.Warnings {
fmt.Fprintf(w, "Warning:\t%s\n", warn)
}
})
},
}
var topN int
summary := &cobra.Command{
Use: "summary",
Short: "Totals, result codes, record types and the top names and clients",
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
var s stats.Summary
if err := c.Get(ctx(), "/stats/summary", withViews(q("since", summarySince, "top", strconv.Itoa(topN))), &s); err != nil {
return err
}
return output(s, func(w io.Writer) {
fmt.Fprintf(w, "Range:\t%s to %s\n", s.From.Local().Format("2006-01-02 15:04"), s.To.Local().Format("2006-01-02 15:04"))
if len(s.FilterViews) > 0 {
since := ""
if s.ViewsSince != nil {
since = ", per view since " + s.ViewsSince.Local().Format("2006-01-02 15:04")
}
fmt.Fprintf(w, "Views:\t%s (from the response log%s; named's own counters are left out)\n", strings.Join(s.FilterViews, ", "), since)
fmt.Fprintf(w, "Answers:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS)
} else {
fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS)
}
if s.Peak != nil {
what := "queries"
if len(s.FilterViews) > 0 {
what = "answers"
}
fmt.Fprintf(w, "Busiest %s:\t%s with %d %s\n", s.Peak.Per, s.Peak.Time.Local().Format("2006-01-02 15:04"), s.Peak.Queries, what)
}
if s.CacheHitRatio != nil {
fmt.Fprintf(w, "Cache hit ratio:\t%.1f%%\n", *s.CacheHitRatio*100)
}
if s.NXDomainRatio != nil {
fmt.Fprintf(w, "NXDOMAIN:\t%.1f%% of queries\n", *s.NXDomainRatio*100)
}
if s.BlockedRatio != nil {
fmt.Fprintf(w, "Blocked:\t%d (%.1f%% of queries): %s\n", s.Blocked, *s.BlockedRatio*100, entryLine(s.Blocklists, s.Blocked, 6))
}
fmt.Fprintf(w, "Result codes:\t%s\n", entryLine(s.RCodes, s.Queries, 6))
fmt.Fprintf(w, "Record types:\t%s\n", entryLine(s.QTypes, s.Queries, 8))
if len(s.Answers) > 0 {
fmt.Fprintf(w, "Answers:\t%s\n", entryLine(s.Answers, s.Queries, 2))
}
if len(s.Views) > 0 {
fmt.Fprintf(w, "Views:\t%s\n", entryLine(s.Views, 0, 10))
}
if len(s.Zones) > 0 {
fmt.Fprintf(w, "Zones:\t%s\n", strings.ReplaceAll(entryLine(s.Zones, 0, 10), stats.OtherZone+" ", "other "))
}
for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain, stats.KindBlocked} {
es := s.Tops[k]
if len(es) == 0 {
continue
}
fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names", stats.KindBlocked: "Top blocked names"}[k])
for _, e := range es {
key := e.Key
if e.Name != "" {
key += " (" + e.Name + ")"
}
fmt.Fprintf(w, " %s\t%d\n", key, e.Count)
}
}
})
},
}
sinceFlag(summary, &summarySince, "24h", sinceHelp)
summary.Flags().IntVar(&topN, "top", 10, "entries per top list")
var limit int
top := &cobra.Command{
Use: "top [names|clients|nxdomain|servfail|refused|blocked|blocked-clients|ratelimited|ratelimited-names]",
Short: "The most queried names, busiest clients, names with errors, refused clients, blocked names",
Example: ` jarl stats top names --since 7d
jarl stats top clients --limit 50
jarl stats top nxdomain # misconfigured hosts and typos
jarl stats top refused # who is asking for what they may not have
jarl stats top blocked # what blocking stops most (docs/blocking.md)
jarl stats top blocked-clients # the devices blocking stops most
jarl stats top ratelimited # client networks rate limiting limits, or would (docs/exposure.md)`,
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
kind := stats.KindName
if len(args) == 1 {
kind = strings.TrimSuffix(strings.ToLower(args[0]), "s")
if kind == "nxdomain" || kind == "servfail" {
kind = strings.ToLower(args[0])
}
if kind == "blocked-client" {
kind = stats.KindBlockedClient
}
if kind == "ratelimited-name" {
kind = stats.KindRateLimitedName
}
}
c, err := newClient()
if err != nil {
return err
}
var es []stats.Entry
if err := c.Get(ctx(), "/stats/top", withViews(q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit))), &es); err != nil {
return err
}
return output(es, func(w io.Writer) {
var total int64
for _, e := range es {
total += e.Count
}
head := map[string]string{stats.KindClient: "CLIENT", stats.KindRefused: "CLIENT", stats.KindBlockedClient: "CLIENT"}[kind]
if head == "" {
head = "NAME"
}
named := slices.ContainsFunc(es, func(e stats.Entry) bool { return e.Name != "" })
if named {
fmt.Fprintf(w, "#\t%s\tHOST\tQUERIES\t\n", head)
} else {
fmt.Fprintf(w, "#\t%s\tQUERIES\t\n", head)
}
for i, e := range es {
if named {
fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%s\n", i+1, e.Key, dash(e.Name), e.Count, bar(e.Count, es[0].Count, 20))
} else {
fmt.Fprintf(w, "%d\t%s\t%d\t%s\n", i+1, e.Key, e.Count, bar(e.Count, es[0].Count, 20))
}
}
})
},
}
sinceFlag(top, &topSince, "24h", sinceHelp)
top.Flags().IntVar(&limit, "limit", 20, "how many")
var step string
var keys []string
var table bool
timeline := &cobra.Command{
Use: "timeline",
Short: "Counters over time (queries by default)",
Example: ` jarl stats timeline --since 6h
jarl stats timeline --since 7d --key rcode:NXDOMAIN
jarl stats timeline --since 24h --key 'qtype:*' -o json`,
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
qv := q("since", timelineSince, "step", step)
for _, k := range keys {
qv.Add("key", k)
}
var tl stats.Timeline
if err := c.Get(ctx(), "/stats/timeline", withViews(qv), &tl); err != nil {
return err
}
return output(tl, func(w io.Writer) {
for _, s := range tl.Series {
var total, peak int64
for _, v := range s.Values {
total += v
peak = max(peak, v)
}
fmt.Fprintf(w, "%s\ttotal %d, peak %d per %s\n", s.Key, total, peak, tl.Step)
fmt.Fprintf(w, "\t%s\n", sparkline(s.Values))
}
if len(tl.Times) > 0 {
fmt.Fprintf(w, "\t%s ... %s\n", time.Unix(tl.Times[0], 0).Local().Format("Jan 2 15:04"), time.Unix(tl.Times[len(tl.Times)-1], 0).Local().Format("Jan 2 15:04"))
}
if !table {
return
}
fmt.Fprint(w, "\nTIME")
for _, s := range tl.Series {
fmt.Fprintf(w, "\t%s", strings.ToUpper(s.Key))
}
fmt.Fprintln(w)
for i, t := range tl.Times {
fmt.Fprint(w, time.Unix(t, 0).Local().Format("2006-01-02 15:04"))
for _, s := range tl.Series {
fmt.Fprintf(w, "\t%d", s.Values[i])
}
fmt.Fprintln(w)
}
})
},
}
sinceFlag(timeline, &timelineSince, "24h", sinceHelp)
timeline.Flags().StringVar(&step, "step", "auto", "bucket size: auto, 1m, 5m, 15m, 1h, 6h, 1d")
timeline.Flags().BoolVar(&table, "table", false, "also print every bucket (use -o json to export)")
timeline.Flags().StringArrayVar(&keys, "key", nil, `counter (repeatable), e.g. queries, rcode:NXDOMAIN, 'qtype:*' (default queries)`)
var hmKey string
heatmap := &cobra.Command{
Use: "heatmap",
Short: "Average queries per hour by weekday and hour of day",
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
var hm stats.Heatmap
if err := c.Get(ctx(), "/stats/heatmap", withViews(q("since", heatmapSince, "key", hmKey)), &hm); err != nil {
return err
}
return output(hm, func(w io.Writer) {
peak := 0.0
for d := range 7 {
for h := range 24 {
peak = math.Max(peak, hm.Avg[d][h])
}
}
fmt.Fprintf(w, "Average %s per hour since %s (darker = busier, · = no data)\n\n", hm.Key, hm.From.Local().Format("2006-01-02"))
fmt.Fprint(w, " ")
for h := 0; h < 24; h += 3 {
fmt.Fprintf(w, "%-6s", fmt.Sprintf("%02d", h))
}
fmt.Fprintln(w)
shades := []string{"░░", "▒▒", "▓▓", "██"}
for d, day := range []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} {
fmt.Fprintf(w, "%s ", day)
for h := range 24 {
v := hm.Avg[d][h]
switch {
case v < 0:
fmt.Fprint(w, "· ")
case peak <= 0:
fmt.Fprint(w, shades[0])
default:
fmt.Fprint(w, shades[min(int(v/peak*float64(len(shades))), len(shades)-1)])
}
}
fmt.Fprintln(w)
}
fmt.Fprintf(w, "\nBusiest slot: %.0f %s per hour on average\n", peak, hm.Key)
})
},
}
sinceFlag(heatmap, &heatmapSince, "28d", "time range (at most the hourly retention)")
heatmap.Flags().StringVar(&hmKey, "key", "queries", "counter to average")
breakdown := &cobra.Command{
Use: "breakdown GROUP",
Short: "Totals by " + strings.Join(stats.Groups, ", "),
Example: ` jarl stats breakdown qtype
jarl stats breakdown zone --since 7d
jarl stats breakdown rtt # upstream response times of recursion`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
var es []stats.Entry
if err := c.Get(ctx(), "/stats/breakdown", withViews(q("group", args[0], "since", breakdownSince)), &es); err != nil {
return err
}
return output(es, func(w io.Writer) {
var total int64
for _, e := range es {
total += e.Count
}
fmt.Fprintf(w, "%s\tCOUNT\tSHARE\t\n", strings.ToUpper(args[0]))
for _, e := range es {
key := e.Key
if args[0] == "zone" && key == stats.OtherZone {
key = "(not a local zone)"
}
fmt.Fprintf(w, "%s\t%d\t%.1f%%\t%s\n", key, e.Count, pct(e.Count, total), bar(e.Count, es[0].Count, 20))
}
})
},
}
sinceFlag(breakdown, &breakdownSince, "24h", sinceHelp)
server := &cobra.Command{
Use: "server",
Short: "Show named's memory, CPU, cache and clients, the host's memory and load, and free disk space",
Long: `Server statistics, sampled once a minute: named's memory and CPU use,
cache size, recursive and TCP clients against their limits, the host's
memory and load, and the space left where zone files, logs and Jarl's
data are. Memory and CPU over time: 'jarl stats timeline --key
g:named:rss,g:named:cpu' (averages; CPU in tenths of a percent).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
var resp struct {
Server *stats.Server `json:"server"`
Named map[string]string `json:"named,omitempty"`
NamedError string `json:"named_error,omitempty"`
Disks []stats.Disk `json:"disks"`
}
if err := c.Get(ctx(), "/stats/server", nil, &resp); err != nil {
return err
}
return output(resp, func(w io.Writer) {
s := resp.Server
if s == nil {
fmt.Fprintln(w, "No sample yet:\tthe first is taken a minute after jarl starts")
return
}
fmt.Fprintf(w, "named:\t%s, up since %s, reconfigured %s\n", s.Version, isoLocal(s.BootTime), isoLocal(s.ConfigTime))
if s.PID > 0 {
fmt.Fprintf(w, "Memory:\t%s resident (named counts %s), %d threads, PID %d\n", sizeStr(s.RSS), sizeStr(s.MemInUse), s.Threads, s.PID)
fmt.Fprintf(w, "CPU:\t%.1f%% of one CPU over the last minute\n", s.CPUPercent)
} else {
fmt.Fprintf(w, "Memory:\tnamed counts %s (its process isn't visible to jarld)\n", sizeStr(s.MemInUse))
}
fmt.Fprintf(w, "Cache:\t%d RRsets, %s\n", s.CacheRRsets, sizeStr(s.CacheMem))
if n := resp.Named; n != nil {
fmt.Fprintf(w, "Recursive clients:\t%s (now/soft limit/limit), most at once %d\n", n["recursive_clients"], s.RecursionHighwater)
fmt.Fprintf(w, "TCP clients:\t%s (now/limit), most at once %d\n", n["tcp_clients"], s.TCPHighwater)
} else if resp.NamedError != "" {
fmt.Fprintf(w, "Clients:\tunknown: %s\n", resp.NamedError)
}
fmt.Fprintf(w, "Sockets:\t%d UDP, %d TCP\n", s.SocketsUDP, s.SocketsTCP)
if h := s.Host; h != nil && h.MemTotal > 0 {
fmt.Fprintf(w, "Host:\t%s of %s memory in use, load %v, %d CPUs, up %s\n", sizeStr(h.MemTotal-h.MemAvailable), sizeStr(h.MemTotal), h.Load, h.CPUs, (time.Duration(h.Uptime) * time.Second).String())
}
for _, d := range resp.Disks {
fmt.Fprintf(w, "Disk:\t%s free of %s for %s (%s)\n", sizeStr(d.Free), sizeStr(d.Total), strings.Join(d.Holds, ", "), d.Path)
}
})
},
}
cmd.AddCommand(status, summary, top, timeline, heatmap, breakdown, server)
return cmd
}
func pct(n, total int64) float64 {
if total == 0 {
return 0
}
return float64(n) / float64(total) * 100
}
// entryLine formats the first n entries as "A 61% · AAAA 30% · ...".
func entryLine(es []stats.Entry, total int64, n int) string {
if total == 0 {
for _, e := range es {
total += e.Count
}
}
var parts []string
for i, e := range es {
if i == n {
parts = append(parts, fmt.Sprintf("+%d more", len(es)-n))
break
}
parts = append(parts, fmt.Sprintf("%s %.1f%%", e.Key, pct(e.Count, total)))
}
if len(parts) == 0 {
return "-"
}
return strings.Join(parts, " · ")
}
func bar(v, peak int64, width int) string {
if peak <= 0 {
return ""
}
n := int(math.Round(float64(v) / float64(peak) * float64(width)))
return strings.Repeat("■", max(n, 1))
}
// sparkline draws values with block characters, at most 120 wide.
func sparkline(vals []int64) string {
if len(vals) == 0 {
return ""
}
if len(vals) > 120 {
// Combine neighbouring buckets.
per := (len(vals) + 119) / 120
var merged []int64
for i := 0; i < len(vals); i += per {
var s int64
for _, v := range vals[i:min(i+per, len(vals))] {
s += v
}
merged = append(merged, s)
}
vals = merged
}
blocks := []rune("▁▂▃▄▅▆▇█")
var peak int64
for _, v := range vals {
peak = max(peak, v)
}
var sb strings.Builder
for _, v := range vals {
if peak == 0 {
sb.WriteRune(blocks[0])
continue
}
sb.WriteRune(blocks[min(int(float64(v)/float64(peak)*float64(len(blocks)-1)+0.5), len(blocks)-1)])
}
return sb.String()
}
// sizeStr formats a size in binary units.
func sizeStr(n int64) string {
units := []string{"B", "KiB", "MiB", "GiB", "TiB"}
f, i := float64(n), 0
for f >= 1024 && i < len(units)-1 {
f /= 1024
i++
}
if i == 0 || f >= 100 {
return fmt.Sprintf("%.0f %s", f, units[i])
}
return fmt.Sprintf("%.1f %s", f, units[i])
}
// isoLocal shows a time named reports (RFC 3339) in local time.
func isoLocal(s string) string {
t, err := time.Parse(time.RFC3339Nano, s)
if err != nil {
return s
}
return t.Local().Format("2006-01-02 15:04")
}