package main import ( "fmt" "io" "math" "net/url" "slices" "strconv" "strings" "time" "github.com/spf13/cobra" "jarl/internal/stats" ) func statsCmd() *cobra.Command { cmd := &cobra.Command{ Use: "stats", Short: "Query statistics: volume over time, top names and clients, result codes", Long: `Query statistics collected from named (see docs/stats.md). They must be enabled in /etc/jarl/jarl.yaml (stats.enabled: true). Ranges are given with --since: 90m, 24h, 7d, 2w. --view limits the figures to some views (repeat it, or separate with commas): answers from named's response log, per view; named's own counters stay for the whole server. Use -o json or -o yaml to export the data.`, } var statsViews []string cmd.PersistentFlags().StringSliceVar(&statsViews, "view", nil, "only these views (repeat, or comma separated)") // withViews adds the --view filter to a query. withViews := func(v url.Values) url.Values { if len(statsViews) > 0 { if v == nil { v = url.Values{} } v.Set("views", strings.Join(statsViews, ",")) } return v } // Each command has its own --since variable: their defaults differ, and // a shared variable would take the last default for all of them. var summarySince, topSince, timelineSince, heatmapSince, breakdownSince string sinceFlag := func(c *cobra.Command, v *string, def, help string) { c.Flags().StringVar(v, "since", def, help) } const sinceHelp = "time range: 90m, 24h, 7d, 2w" status := &cobra.Command{ Use: "status", Short: "Show whether statistics are collected and any problems", RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } var st stats.Status if err := c.Get(ctx(), "/stats/status", nil, &st); err != nil { return err } return output(st, func(w io.Writer) { if !st.Enabled { fmt.Fprintln(w, "Statistics:\tdisabled (set stats.enabled: true in /etc/jarl/jarl.yaml and restart jarl)") return } mode := "counters and response log" if !st.QueryLog { mode = "counters only" } fmt.Fprintf(w, "Statistics:\tenabled (%s)\n", mode) fmt.Fprintf(w, "Statistics channel:\t%s, last read %s\n", st.StatisticsChannel, fmtTime(st.LastPoll)) if st.QueryLog { fmt.Fprintf(w, "Response log:\t%s, %d lines read, last %s\n", st.LogFile, st.LinesRead, fmtTime(st.LastLine)) } fmt.Fprintf(w, "Data since:\t%s\n", fmtTime(st.FirstData)) fmt.Fprintf(w, "Retention:\tminutes %s, hours %s, days %s\n", st.Retention.Minute, st.Retention.Hour, st.Retention.Day) for _, warn := range st.Warnings { fmt.Fprintf(w, "Warning:\t%s\n", warn) } }) }, } var topN int summary := &cobra.Command{ Use: "summary", Short: "Totals, result codes, record types and the top names and clients", RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } var s stats.Summary if err := c.Get(ctx(), "/stats/summary", withViews(q("since", summarySince, "top", strconv.Itoa(topN))), &s); err != nil { return err } return output(s, func(w io.Writer) { fmt.Fprintf(w, "Range:\t%s to %s\n", s.From.Local().Format("2006-01-02 15:04"), s.To.Local().Format("2006-01-02 15:04")) if len(s.FilterViews) > 0 { since := "" if s.ViewsSince != nil { since = ", per view since " + s.ViewsSince.Local().Format("2006-01-02 15:04") } fmt.Fprintf(w, "Views:\t%s (from the response log%s; named's own counters are left out)\n", strings.Join(s.FilterViews, ", "), since) fmt.Fprintf(w, "Answers:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) } else { fmt.Fprintf(w, "Queries:\t%d (%.2f per second on average)\n", s.Queries, s.AvgQPS) } if s.Peak != nil { what := "queries" if len(s.FilterViews) > 0 { what = "answers" } fmt.Fprintf(w, "Busiest %s:\t%s with %d %s\n", s.Peak.Per, s.Peak.Time.Local().Format("2006-01-02 15:04"), s.Peak.Queries, what) } if s.CacheHitRatio != nil { fmt.Fprintf(w, "Cache hit ratio:\t%.1f%%\n", *s.CacheHitRatio*100) } if s.NXDomainRatio != nil { fmt.Fprintf(w, "NXDOMAIN:\t%.1f%% of queries\n", *s.NXDomainRatio*100) } if s.BlockedRatio != nil { fmt.Fprintf(w, "Blocked:\t%d (%.1f%% of queries): %s\n", s.Blocked, *s.BlockedRatio*100, entryLine(s.Blocklists, s.Blocked, 6)) } fmt.Fprintf(w, "Result codes:\t%s\n", entryLine(s.RCodes, s.Queries, 6)) fmt.Fprintf(w, "Record types:\t%s\n", entryLine(s.QTypes, s.Queries, 8)) if len(s.Answers) > 0 { fmt.Fprintf(w, "Answers:\t%s\n", entryLine(s.Answers, s.Queries, 2)) } if len(s.Views) > 0 { fmt.Fprintf(w, "Views:\t%s\n", entryLine(s.Views, 0, 10)) } if len(s.Zones) > 0 { fmt.Fprintf(w, "Zones:\t%s\n", strings.ReplaceAll(entryLine(s.Zones, 0, 10), stats.OtherZone+" ", "other ")) } for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain, stats.KindBlocked} { es := s.Tops[k] if len(es) == 0 { continue } fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names", stats.KindBlocked: "Top blocked names"}[k]) for _, e := range es { key := e.Key if e.Name != "" { key += " (" + e.Name + ")" } fmt.Fprintf(w, " %s\t%d\n", key, e.Count) } } }) }, } sinceFlag(summary, &summarySince, "24h", sinceHelp) summary.Flags().IntVar(&topN, "top", 10, "entries per top list") var limit int top := &cobra.Command{ Use: "top [names|clients|nxdomain|servfail|refused|blocked|blocked-clients|ratelimited|ratelimited-names]", Short: "The most queried names, busiest clients, names with errors, refused clients, blocked names", Example: ` jarl stats top names --since 7d jarl stats top clients --limit 50 jarl stats top nxdomain # misconfigured hosts and typos jarl stats top refused # who is asking for what they may not have jarl stats top blocked # what blocking stops most (docs/blocking.md) jarl stats top blocked-clients # the devices blocking stops most jarl stats top ratelimited # client networks rate limiting limits, or would (docs/exposure.md)`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { kind := stats.KindName if len(args) == 1 { kind = strings.TrimSuffix(strings.ToLower(args[0]), "s") if kind == "nxdomain" || kind == "servfail" { kind = strings.ToLower(args[0]) } if kind == "blocked-client" { kind = stats.KindBlockedClient } if kind == "ratelimited-name" { kind = stats.KindRateLimitedName } } c, err := newClient() if err != nil { return err } var es []stats.Entry if err := c.Get(ctx(), "/stats/top", withViews(q("kind", kind, "since", topSince, "limit", strconv.Itoa(limit))), &es); err != nil { return err } return output(es, func(w io.Writer) { var total int64 for _, e := range es { total += e.Count } head := map[string]string{stats.KindClient: "CLIENT", stats.KindRefused: "CLIENT", stats.KindBlockedClient: "CLIENT"}[kind] if head == "" { head = "NAME" } named := slices.ContainsFunc(es, func(e stats.Entry) bool { return e.Name != "" }) if named { fmt.Fprintf(w, "#\t%s\tHOST\tQUERIES\t\n", head) } else { fmt.Fprintf(w, "#\t%s\tQUERIES\t\n", head) } for i, e := range es { if named { fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%s\n", i+1, e.Key, dash(e.Name), e.Count, bar(e.Count, es[0].Count, 20)) } else { fmt.Fprintf(w, "%d\t%s\t%d\t%s\n", i+1, e.Key, e.Count, bar(e.Count, es[0].Count, 20)) } } }) }, } sinceFlag(top, &topSince, "24h", sinceHelp) top.Flags().IntVar(&limit, "limit", 20, "how many") var step string var keys []string var table bool timeline := &cobra.Command{ Use: "timeline", Short: "Counters over time (queries by default)", Example: ` jarl stats timeline --since 6h jarl stats timeline --since 7d --key rcode:NXDOMAIN jarl stats timeline --since 24h --key 'qtype:*' -o json`, RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } qv := q("since", timelineSince, "step", step) for _, k := range keys { qv.Add("key", k) } var tl stats.Timeline if err := c.Get(ctx(), "/stats/timeline", withViews(qv), &tl); err != nil { return err } return output(tl, func(w io.Writer) { for _, s := range tl.Series { var total, peak int64 for _, v := range s.Values { total += v peak = max(peak, v) } fmt.Fprintf(w, "%s\ttotal %d, peak %d per %s\n", s.Key, total, peak, tl.Step) fmt.Fprintf(w, "\t%s\n", sparkline(s.Values)) } if len(tl.Times) > 0 { fmt.Fprintf(w, "\t%s ... %s\n", time.Unix(tl.Times[0], 0).Local().Format("Jan 2 15:04"), time.Unix(tl.Times[len(tl.Times)-1], 0).Local().Format("Jan 2 15:04")) } if !table { return } fmt.Fprint(w, "\nTIME") for _, s := range tl.Series { fmt.Fprintf(w, "\t%s", strings.ToUpper(s.Key)) } fmt.Fprintln(w) for i, t := range tl.Times { fmt.Fprint(w, time.Unix(t, 0).Local().Format("2006-01-02 15:04")) for _, s := range tl.Series { fmt.Fprintf(w, "\t%d", s.Values[i]) } fmt.Fprintln(w) } }) }, } sinceFlag(timeline, &timelineSince, "24h", sinceHelp) timeline.Flags().StringVar(&step, "step", "auto", "bucket size: auto, 1m, 5m, 15m, 1h, 6h, 1d") timeline.Flags().BoolVar(&table, "table", false, "also print every bucket (use -o json to export)") timeline.Flags().StringArrayVar(&keys, "key", nil, `counter (repeatable), e.g. queries, rcode:NXDOMAIN, 'qtype:*' (default queries)`) var hmKey string heatmap := &cobra.Command{ Use: "heatmap", Short: "Average queries per hour by weekday and hour of day", RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } var hm stats.Heatmap if err := c.Get(ctx(), "/stats/heatmap", withViews(q("since", heatmapSince, "key", hmKey)), &hm); err != nil { return err } return output(hm, func(w io.Writer) { peak := 0.0 for d := range 7 { for h := range 24 { peak = math.Max(peak, hm.Avg[d][h]) } } fmt.Fprintf(w, "Average %s per hour since %s (darker = busier, · = no data)\n\n", hm.Key, hm.From.Local().Format("2006-01-02")) fmt.Fprint(w, " ") for h := 0; h < 24; h += 3 { fmt.Fprintf(w, "%-6s", fmt.Sprintf("%02d", h)) } fmt.Fprintln(w) shades := []string{"░░", "▒▒", "▓▓", "██"} for d, day := range []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} { fmt.Fprintf(w, "%s ", day) for h := range 24 { v := hm.Avg[d][h] switch { case v < 0: fmt.Fprint(w, "· ") case peak <= 0: fmt.Fprint(w, shades[0]) default: fmt.Fprint(w, shades[min(int(v/peak*float64(len(shades))), len(shades)-1)]) } } fmt.Fprintln(w) } fmt.Fprintf(w, "\nBusiest slot: %.0f %s per hour on average\n", peak, hm.Key) }) }, } sinceFlag(heatmap, &heatmapSince, "28d", "time range (at most the hourly retention)") heatmap.Flags().StringVar(&hmKey, "key", "queries", "counter to average") breakdown := &cobra.Command{ Use: "breakdown GROUP", Short: "Totals by " + strings.Join(stats.Groups, ", "), Example: ` jarl stats breakdown qtype jarl stats breakdown zone --since 7d jarl stats breakdown rtt # upstream response times of recursion`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } var es []stats.Entry if err := c.Get(ctx(), "/stats/breakdown", withViews(q("group", args[0], "since", breakdownSince)), &es); err != nil { return err } return output(es, func(w io.Writer) { var total int64 for _, e := range es { total += e.Count } fmt.Fprintf(w, "%s\tCOUNT\tSHARE\t\n", strings.ToUpper(args[0])) for _, e := range es { key := e.Key if args[0] == "zone" && key == stats.OtherZone { key = "(not a local zone)" } fmt.Fprintf(w, "%s\t%d\t%.1f%%\t%s\n", key, e.Count, pct(e.Count, total), bar(e.Count, es[0].Count, 20)) } }) }, } sinceFlag(breakdown, &breakdownSince, "24h", sinceHelp) server := &cobra.Command{ Use: "server", Short: "Show named's memory, CPU, cache and clients, the host's memory and load, and free disk space", Long: `Server statistics, sampled once a minute: named's memory and CPU use, cache size, recursive and TCP clients against their limits, the host's memory and load, and the space left where zone files, logs and Jarl's data are. Memory and CPU over time: 'jarl stats timeline --key g:named:rss,g:named:cpu' (averages; CPU in tenths of a percent).`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { c, err := newClient() if err != nil { return err } var resp struct { Server *stats.Server `json:"server"` Named map[string]string `json:"named,omitempty"` NamedError string `json:"named_error,omitempty"` Disks []stats.Disk `json:"disks"` } if err := c.Get(ctx(), "/stats/server", nil, &resp); err != nil { return err } return output(resp, func(w io.Writer) { s := resp.Server if s == nil { fmt.Fprintln(w, "No sample yet:\tthe first is taken a minute after jarl starts") return } fmt.Fprintf(w, "named:\t%s, up since %s, reconfigured %s\n", s.Version, isoLocal(s.BootTime), isoLocal(s.ConfigTime)) if s.PID > 0 { fmt.Fprintf(w, "Memory:\t%s resident (named counts %s), %d threads, PID %d\n", sizeStr(s.RSS), sizeStr(s.MemInUse), s.Threads, s.PID) fmt.Fprintf(w, "CPU:\t%.1f%% of one CPU over the last minute\n", s.CPUPercent) } else { fmt.Fprintf(w, "Memory:\tnamed counts %s (its process isn't visible to jarld)\n", sizeStr(s.MemInUse)) } fmt.Fprintf(w, "Cache:\t%d RRsets, %s\n", s.CacheRRsets, sizeStr(s.CacheMem)) if n := resp.Named; n != nil { fmt.Fprintf(w, "Recursive clients:\t%s (now/soft limit/limit), most at once %d\n", n["recursive_clients"], s.RecursionHighwater) fmt.Fprintf(w, "TCP clients:\t%s (now/limit), most at once %d\n", n["tcp_clients"], s.TCPHighwater) } else if resp.NamedError != "" { fmt.Fprintf(w, "Clients:\tunknown: %s\n", resp.NamedError) } fmt.Fprintf(w, "Sockets:\t%d UDP, %d TCP\n", s.SocketsUDP, s.SocketsTCP) if h := s.Host; h != nil && h.MemTotal > 0 { fmt.Fprintf(w, "Host:\t%s of %s memory in use, load %v, %d CPUs, up %s\n", sizeStr(h.MemTotal-h.MemAvailable), sizeStr(h.MemTotal), h.Load, h.CPUs, (time.Duration(h.Uptime) * time.Second).String()) } for _, d := range resp.Disks { fmt.Fprintf(w, "Disk:\t%s free of %s for %s (%s)\n", sizeStr(d.Free), sizeStr(d.Total), strings.Join(d.Holds, ", "), d.Path) } }) }, } cmd.AddCommand(status, summary, top, timeline, heatmap, breakdown, server) return cmd } func pct(n, total int64) float64 { if total == 0 { return 0 } return float64(n) / float64(total) * 100 } // entryLine formats the first n entries as "A 61% · AAAA 30% · ...". func entryLine(es []stats.Entry, total int64, n int) string { if total == 0 { for _, e := range es { total += e.Count } } var parts []string for i, e := range es { if i == n { parts = append(parts, fmt.Sprintf("+%d more", len(es)-n)) break } parts = append(parts, fmt.Sprintf("%s %.1f%%", e.Key, pct(e.Count, total))) } if len(parts) == 0 { return "-" } return strings.Join(parts, " · ") } func bar(v, peak int64, width int) string { if peak <= 0 { return "" } n := int(math.Round(float64(v) / float64(peak) * float64(width))) return strings.Repeat("■", max(n, 1)) } // sparkline draws values with block characters, at most 120 wide. func sparkline(vals []int64) string { if len(vals) == 0 { return "" } if len(vals) > 120 { // Combine neighbouring buckets. per := (len(vals) + 119) / 120 var merged []int64 for i := 0; i < len(vals); i += per { var s int64 for _, v := range vals[i:min(i+per, len(vals))] { s += v } merged = append(merged, s) } vals = merged } blocks := []rune("▁▂▃▄▅▆▇█") var peak int64 for _, v := range vals { peak = max(peak, v) } var sb strings.Builder for _, v := range vals { if peak == 0 { sb.WriteRune(blocks[0]) continue } sb.WriteRune(blocks[min(int(float64(v)/float64(peak)*float64(len(blocks)-1)+0.5), len(blocks)-1)]) } return sb.String() } // sizeStr formats a size in binary units. func sizeStr(n int64) string { units := []string{"B", "KiB", "MiB", "GiB", "TiB"} f, i := float64(n), 0 for f >= 1024 && i < len(units)-1 { f /= 1024 i++ } if i == 0 || f >= 100 { return fmt.Sprintf("%.0f %s", f, units[i]) } return fmt.Sprintf("%.1f %s", f, units[i]) } // isoLocal shows a time named reports (RFC 3339) in local time. func isoLocal(s string) string { t, err := time.Parse(time.RFC3339Nano, s) if err != nil { return s } return t.Local().Format("2006-01-02 15:04") }