housecarl/cmd/hc/stats.go

package main

import (
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"
)

// Statistics: the figures housecarld keeps over time (docs/stats.md). The
// types mirror internal/stats's JSON, so hc doesn't link the database.

type statEntry struct {
	Key   string `json:"key"`
	Count int64  `json:"count"`
	Name  string `json:"name,omitempty"`
}

type statTraffic struct {
	Name     string `json:"name"`
	Received int64  `json:"received"`
	Sent     int64  `json:"sent"`
}

type statSummary struct {
	Since           string        `json:"since"`
	From            time.Time     `json:"from"`
	Zones           []statTraffic `json:"zones"`
	Interfaces      []statTraffic `json:"interfaces"`
	Drops           int64         `json:"drops"`
	DropsByChain    []statEntry   `json:"drops_by_chain"`
	Rules           []statEntry   `json:"rules"`
	PeakConnections int64         `json:"peak_connections"`
	ConntrackMax    int64         `json:"conntrack_max"`
	BlockedSources  []statEntry   `json:"blocked_sources"`
	BlockedPorts    []statEntry   `json:"blocked_ports"`
	DevicesUp       []statEntry   `json:"devices_up"`
	DevicesDown     []statEntry   `json:"devices_down"`
}

type statTimeline struct {
	Step   string  `json:"step"`
	Times  []int64 `json:"times"`
	Series []struct {
		Key    string  `json:"key"`
		Values []int64 `json:"values"`
	} `json:"series"`
}

type statStatus struct {
	Enabled     bool       `json:"enabled"`
	Database    string     `json:"database"`
	Size        int64      `json:"size"`
	LastSample  *time.Time `json:"last_sample"`
	SampleError string     `json:"sample_error"`
	FirstData   *time.Time `json:"first_data"`
	PerDevice   bool       `json:"per_device"`
	Retention   struct {
		Minute, Hour, Day string
	} `json:"retention"`
	Latest *struct {
		Conntrack    int64   `json:"conntrack"`
		ConntrackMax int64   `json:"conntrack_max"`
		RSS          int64   `json:"rss"`
		CPUPermille  int64   `json:"cpu_permille"`
		MemUsed      int64   `json:"mem_used"`
		Load         float64 `json:"load"`
	} `json:"latest"`
}

func statsCmd() *cobra.Command {
	var since string
	var limit int
	cmd := &cobra.Command{
		Use: "stats", Short: "Traffic, drops, blocked sources and devices over time",
		Long: `Sums up what the firewall did over a while (--since: 90m, 24h, 7d, 4w):
traffic per zone, packets dropped, the busiest rules, the most connections
at once, the sources blocked most and what they tried, and (with traffic
per device on) the devices that used the most.

  hc stats --since 7d
  hc stats traffic --since 24h
  hc stats top devices --since 30d
  hc stats rules --unused`,
		Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			s, err := get[statSummary]("/stats/summary", q("since", since, "limit", strconv.Itoa(limit)))
			if err != nil {
				return err
			}
			return output(s, func(w io.Writer) { printSummary(w, s) })
		},
	}
	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d, 4w…")
	cmd.Flags().IntVar(&limit, "limit", 5, "entries per top list")
	cmd.AddCommand(statsStatusCmd(), statsTrafficCmd(), statsTopCmd(), statsRulesCmd(), statsConnectionsCmd())
	return cmd
}

func printSummary(w io.Writer, s statSummary) {
	fmt.Fprintf(w, "Since %s (%s)\n\n", s.From.Local().Format("2006-01-02 15:04"), s.Since)
	if len(s.Zones) > 0 {
		fmt.Fprintln(w, "ZONE\tRECEIVED\tSENT")
		for _, t := range s.Zones {
			fmt.Fprintf(w, "%s\t%s\t%s\n", t.Name, humanBytes(t.Received), humanBytes(t.Sent))
		}
		fmt.Fprintln(w)
	}
	var chains []string
	for _, e := range s.DropsByChain {
		chains = append(chains, fmt.Sprintf("%s %s", e.Key, humanCount(uint64(e.Count))))
	}
	drops := humanCount(uint64(s.Drops))
	if len(chains) > 0 {
		drops += " (" + strings.Join(chains, ", ") + ")"
	}
	fmt.Fprintf(w, "Dropped:\t%s packets\n", drops)
	if s.ConntrackMax > 0 {
		fmt.Fprintf(w, "Connections:\tat most %d at once (of %d)\n", s.PeakConnections, s.ConntrackMax)
	}
	list := func(title string, es []statEntry, unit func(int64) string) {
		if len(es) == 0 {
			return
		}
		fmt.Fprintf(w, "\n%s\n", title)
		for _, e := range es {
			name := e.Key
			if e.Name != "" {
				name = e.Name
			}
			fmt.Fprintf(w, "  %s\t%s\n", name, unit(e.Count))
		}
	}
	packets := func(n int64) string { return humanCount(uint64(n)) + " packets" }
	list("Busiest rules:", s.Rules, packets)
	list("Blocked most (logged drops):", s.BlockedSources, packets)
	list("What they tried:", s.BlockedPorts, packets)
	list("Devices, sent:", s.DevicesUp, humanBytes)
	list("Devices, received:", s.DevicesDown, humanBytes)
}

func statsStatusCmd() *cobra.Command {
	return &cobra.Command{
		Use: "status", Short: "Whether statistics are collected, since when, and the database", Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			s, err := get[statStatus]("/stats/status", nil)
			if err != nil {
				return err
			}
			return output(s, func(w io.Writer) {
				if !s.Enabled {
					fmt.Fprintln(w, "Statistics are off (stats.enabled in housecarl.yaml).")
					return
				}
				fmt.Fprintf(w, "Collecting:\tyes\n")
				if s.Database != "" {
					fmt.Fprintf(w, "Database:\t%s (%s)\n", s.Database, humanBytes(s.Size))
				} else {
					fmt.Fprintf(w, "Database size:\t%s\n", humanBytes(s.Size))
				}
				if s.FirstData != nil {
					fmt.Fprintf(w, "Since:\t%s\n", s.FirstData.Local().Format("2006-01-02 15:04"))
				}
				if s.LastSample != nil {
					fmt.Fprintf(w, "Last sample:\t%s\n", s.LastSample.Local().Format("15:04:05"))
				}
				if s.SampleError != "" {
					fmt.Fprintf(w, "Error:\t%s\n", s.SampleError)
				}
				fmt.Fprintf(w, "Kept:\tper minute %s, hourly %s, daily %s\n", s.Retention.Minute, s.Retention.Hour, s.Retention.Day)
				fmt.Fprintf(w, "Per device:\t%s\n", map[bool]string{true: "on", false: "off (hc settings set --stats-per-device on)"}[s.PerDevice])
				if l := s.Latest; l != nil {
					if l.ConntrackMax > 0 {
						fmt.Fprintf(w, "Connections now:\t%d of %d\n", l.Conntrack, l.ConntrackMax)
					}
					fmt.Fprintf(w, "housecarld:\t%s, CPU %.1f%%\n", humanBytes(l.RSS), float64(l.CPUPermille)/10)
					fmt.Fprintf(w, "Host:\t%s in use, load %.2f\n", humanBytes(l.MemUsed), l.Load)
				}
			})
		},
	}
}

func statsTrafficCmd() *cobra.Command {
	var since, step string
	var ifaces bool
	cmd := &cobra.Command{
		Use: "traffic", Short: "Bytes received from and sent to each zone (or interface) over time", Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			rx, tx := "zrx", "ztx"
			if ifaces {
				rx, tx = "rx", "tx"
			}
			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", rx+":*,"+tx+":*"))
			if err != nil {
				return err
			}
			return output(tl, func(w io.Writer) { printTimeline(w, tl, rx, tx) })
		},
	}
	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
	cmd.Flags().BoolVar(&ifaces, "interfaces", false, "per interface instead of per zone")
	return cmd
}

// printTimeline writes a row per bucket: each name's received and sent.
func printTimeline(w io.Writer, tl statTimeline, rx, tx string) {
	var names []string
	vals := map[string][]int64{}
	for _, s := range tl.Series {
		vals[s.Key] = s.Values
		if n, ok := strings.CutPrefix(s.Key, rx+":"); ok {
			names = append(names, n)
		}
	}
	for _, s := range tl.Series {
		if n, ok := strings.CutPrefix(s.Key, tx+":"); ok && vals[rx+":"+n] == nil {
			names = append(names, n)
		}
	}
	if len(names) == 0 {
		fmt.Fprintln(w, "no traffic recorded yet")
		return
	}
	fmt.Fprint(w, "TIME")
	for _, n := range names {
		fmt.Fprintf(w, "\t%s ↓\t%s ↑", n, n)
	}
	fmt.Fprintln(w)
	layout := "01-02 15:04"
	if tl.Step == "1d" {
		layout = "2006-01-02"
	}
	at := func(k string, i int) int64 {
		if v := vals[k]; i < len(v) {
			return v[i]
		}
		return 0
	}
	for i, t := range tl.Times {
		fmt.Fprint(w, time.Unix(t, 0).Local().Format(layout))
		for _, n := range names {
			fmt.Fprintf(w, "\t%s\t%s", humanBytes(at(rx+":"+n, i)), humanBytes(at(tx+":"+n, i)))
		}
		fmt.Fprintln(w)
	}
}

func statsTopCmd() *cobra.Command {
	var since string
	var limit int
	cmd := &cobra.Command{
		Use: "top devices|blocked|tried|quota RULE-ID", Short: "The largest: devices by traffic, sources blocked, what they tried, a quota's users",
		Long: `Top lists over the last while:

  devices   inside addresses by traffic to and from external zones (needs
            hc settings set --stats-per-device on)
  blocked   sources of logged drops (the policies' drops with 'log drops'
            on, and rules that log)
  tried     what those drops tried: tcp/22, udp/53, icmp/echo-request
  quota ID  who used a per-address quota of rule ID`,
		Args: cobra.RangeArgs(1, 2),
		RunE: func(cmd *cobra.Command, args []string) error {
			get1 := func(kind string) ([]statEntry, error) {
				return get[[]statEntry]("/stats/top", q("kind", kind, "since", since, "limit", strconv.Itoa(limit)))
			}
			switch args[0] {
			case "devices":
				up, err := get1("device_up")
				if err != nil {
					return err
				}
				down, err := get1("device_down")
				if err != nil {
					return err
				}
				out := map[string][2]int64{}
				var order []string
				for _, l := range [][]statEntry{down, up} {
					for _, e := range l {
						if _, ok := out[e.Key]; !ok {
							order = append(order, e.Key)
						}
					}
				}
				for _, e := range down {
					v := out[e.Key]
					v[0] = e.Count
					out[e.Key] = v
				}
				for _, e := range up {
					v := out[e.Key]
					v[1] = e.Count
					out[e.Key] = v
				}
				return output(map[string]any{"up": up, "down": down}, func(w io.Writer) {
					fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT")
					for _, k := range order {
						fmt.Fprintf(w, "%s\t%s\t%s\n", k, humanBytes(out[k][0]), humanBytes(out[k][1]))
					}
				})
			case "blocked", "tried":
				kind := map[string]string{"blocked": "blocked_src", "tried": "blocked_port"}[args[0]]
				es, err := get1(kind)
				if err != nil {
					return err
				}
				return output(es, func(w io.Writer) {
					fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS")
					for _, e := range es {
						fmt.Fprintf(w, "%s\t%d\n", e.Key, e.Count)
					}
				})
			case "quota":
				if len(args) != 2 {
					return fmt.Errorf("give the rule's ID: hc stats top quota 12")
				}
				es, err := get1("quota_r" + args[1])
				if err != nil {
					return err
				}
				return output(es, func(w io.Writer) {
					fmt.Fprintln(w, "ADDRESS\tUSED")
					for _, e := range es {
						fmt.Fprintf(w, "%s\t%s\n", e.Key, humanBytes(e.Count))
					}
				})
			}
			return fmt.Errorf("unknown list %q: devices, blocked, tried or quota", args[0])
		},
	}
	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
	cmd.Flags().IntVar(&limit, "limit", 20, "at most this many")
	return cmd
}

func statsRulesCmd() *cobra.Command {
	var since string
	var unused bool
	cmd := &cobra.Command{
		Use: "rules", Short: "Packets each rule, built-in and policy counted; --unused: rules that matched nothing", Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			es, err := get[[]statEntry]("/stats/hits", q("since", since))
			if err != nil {
				return err
			}
			if unused {
				var keep []statEntry
				for _, e := range es {
					if e.Count == 0 && strings.HasPrefix(e.Key, "r") {
						keep = append(keep, e)
					}
				}
				es = keep
			}
			return output(es, func(w io.Writer) {
				if unused && len(es) == 0 {
					fmt.Fprintf(w, "every rule matched something in the last %s\n", since)
					return
				}
				fmt.Fprintln(w, "WHAT\tPACKETS")
				for _, e := range es {
					fmt.Fprintf(w, "%s\t%d\n", orDash(e.Name), e.Count)
				}
			})
		},
	}
	cmd.Flags().StringVar(&since, "since", "30d", "the last 24h, 7d, 30d…")
	cmd.Flags().BoolVar(&unused, "unused", false, "only the rules that matched nothing")
	return cmd
}

func statsConnectionsCmd() *cobra.Command {
	var since, step string
	cmd := &cobra.Command{
		Use: "connections", Short: "How many connections the firewall tracked over time, against the limit", Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", "g:conntrack,g:conntrack_max"))
			if err != nil {
				return err
			}
			return output(tl, func(w io.Writer) {
				fmt.Fprintln(w, "TIME\tCONNECTIONS\tLIMIT")
				for i, t := range tl.Times {
					n, m := tl.Series[0].Values[i], tl.Series[1].Values[i]
					if n < 0 {
						continue // no sample
					}
					fmt.Fprintf(w, "%s\t%d\t%d\n", time.Unix(t, 0).Local().Format("01-02 15:04"), n, m)
				}
			})
		},
	}
	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
	return cmd
}