Statistics: traffic, drops, devices and connections over time

internal/stats samples every minute what the kernel already counts: the
interfaces' counters (per zone too), Housecarl's table (hits per rule,
built-in and policy; drops by chain; quotas; per-device sets), the
connection tracking table, and housecarld's and the host's memory, CPU
and load. Logged drops feed top lists of sources and what they tried.
Aggregates go to a database of their own (stats.db; per minute 7 days,
per hour 90 days, per day 2 years), with the store and queries from
Jarl's.

Traffic per inside device is a setting (stats_per_device): counting sets
stats_up/down_v4/v6 and rules at the top of the forward chain, IPv6 per
/64, carried across reloads. Set names starting with meter_, conn_,
quota_ or stats_ are reserved (set_name_reserved).

GET /stats/{status,summary,timeline,top,hits,heatmap}, hc stats, a
Statistics page with Jarl's charts, two charts on the dashboard, and a
conntrack_full warning. e2e: 188 checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/hc/firewall.go

@@ -125,6 +125,7 @@ func settingsCmd() *cobra.Command {
 				fmt.Fprintf(w, "Antispoof IPv6:\t%s\n", list(s.AntispoofV6))
 				fmt.Fprintf(w, "Flowtable:\t%s\n", list(s.FlowtableDevices))
 				fmt.Fprintf(w, "Probation:\t%s\n", fmtSeconds(s.Probation()))
+				fmt.Fprintf(w, "Traffic per device:\t%s\n", map[bool]string{true: "counted (statistics)", false: "not counted"}[s.StatsPerDevice])
 				if s.ManualConfirm {
 					fmt.Fprintln(w, "Confirmation:\tby a person ('hc confirm', the web UI's banner)")
 				} else {
@@ -133,7 +134,7 @@ func settingsCmd() *cobra.Command {
 			})
 		},
 	})
-	var input, forward, output, logDrops, probation, autoConf string
+	var input, forward, output, logDrops, probation, autoConf, perDevice string
 	var builtins, own, flowtable, spoof4, spoof6 []string
 	set := &cobra.Command{
 		Use: "set [flags]", Short: "Change settings (only the flags given)", Args: cobra.NoArgs,
@@ -143,7 +144,8 @@ dhcpv6_client, antispoof, mss_clamp.
 
   hc settings set --builtin mss_clamp=on --own-prefix 2001:db8::/48
   hc settings set --forward-policy drop --log-drops on
-  hc settings set --flowtable wan --flowtable eth1`,
+  hc settings set --flowtable wan --flowtable eth1
+  hc settings set --stats-per-device on`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			fl := cmd.Flags()
 			p := patch{}
@@ -177,6 +179,13 @@ dhcpv6_client, antispoof, mss_clamp.
 					p[map[string]string{"own-prefix": "own_prefixes", "flowtable": "flowtable_devices", "antispoof-v4": "antispoof_v4", "antispoof-v6": "antispoof_v6"}[flag]] = v
 				}
 			}
+			if fl.Changed("stats-per-device") {
+				on, err := onOff("stats-per-device", perDevice)
+				if err != nil {
+					return err
+				}
+				p["stats_per_device"] = on
+			}
 			if fl.Changed("auto-confirm") {
 				on, err := onOff("auto-confirm", autoConf)
 				if err != nil {
@@ -205,6 +214,7 @@ dhcpv6_client, antispoof, mss_clamp.
 	f.StringArrayVar(&spoof4, "antispoof-v4", nil, "IPv4 sources dropped from the internet (replaces)")
 	f.StringArrayVar(&spoof6, "antispoof-v6", nil, "IPv6 sources dropped from the internet (replaces)")
 	f.StringVar(&probation, "probation", "", "how long a network change has to be confirmed (2m)")
+	f.StringVar(&perDevice, "stats-per-device", "", "count traffic per inside device, for the statistics' top devices: on or off")
 	f.StringVar(&autoConf, "auto-confirm", "", "on: the client that made a change confirms it once it reaches Housecarl again over a new connection; off: a person confirms")
 	cmd.AddCommand(set)
 	return cmd

cmd/hc/main.go

@@ -191,7 +191,7 @@ Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found,
 		}
 	}
 	add("fw", statusCmd(), zoneCmd(), setCmd(), serviceCmd(), ruleCmd(), forwardCmd(), snatCmd(), redirectCmd(), settingsCmd(),
-		rulesetCmd(), countersCmd(), quotasCmd(), logCmd(), connectionsCmd(), traceCmd(), matrixCmd(), interfaceCmd())
+		rulesetCmd(), countersCmd(), quotasCmd(), logCmd(), connectionsCmd(), traceCmd(), matrixCmd(), statsCmd(), interfaceCmd())
 	add("changes", changesetCmd(), confirmCmd(), rollbackCmd(), historyCmd(), importCmd(), takeoverCmd())
 	add("admin", whoamiCmd(), userCmd(), tokenCmd(), passwordCmd(), certCmd(), auditCmd(), configCmd())
 	return root

cmd/hc/stats.go

@@ -0,0 +1,401 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+// Statistics: the figures housecarld keeps over time (docs/stats.md). The
+// types mirror internal/stats's JSON, so hc doesn't link the database.
+
+type statEntry struct {
+	Key   string `json:"key"`
+	Count int64  `json:"count"`
+	Name  string `json:"name,omitempty"`
+}
+
+type statTraffic struct {
+	Name     string `json:"name"`
+	Received int64  `json:"received"`
+	Sent     int64  `json:"sent"`
+}
+
+type statSummary struct {
+	Since           string        `json:"since"`
+	From            time.Time     `json:"from"`
+	Zones           []statTraffic `json:"zones"`
+	Interfaces      []statTraffic `json:"interfaces"`
+	Drops           int64         `json:"drops"`
+	DropsByChain    []statEntry   `json:"drops_by_chain"`
+	Rules           []statEntry   `json:"rules"`
+	PeakConnections int64         `json:"peak_connections"`
+	ConntrackMax    int64         `json:"conntrack_max"`
+	BlockedSources  []statEntry   `json:"blocked_sources"`
+	BlockedPorts    []statEntry   `json:"blocked_ports"`
+	DevicesUp       []statEntry   `json:"devices_up"`
+	DevicesDown     []statEntry   `json:"devices_down"`
+}
+
+type statTimeline struct {
+	Step   string  `json:"step"`
+	Times  []int64 `json:"times"`
+	Series []struct {
+		Key    string  `json:"key"`
+		Values []int64 `json:"values"`
+	} `json:"series"`
+}
+
+type statStatus struct {
+	Enabled     bool       `json:"enabled"`
+	Database    string     `json:"database"`
+	Size        int64      `json:"size"`
+	LastSample  *time.Time `json:"last_sample"`
+	SampleError string     `json:"sample_error"`
+	FirstData   *time.Time `json:"first_data"`
+	PerDevice   bool       `json:"per_device"`
+	Retention   struct {
+		Minute, Hour, Day string
+	} `json:"retention"`
+	Latest *struct {
+		Conntrack    int64   `json:"conntrack"`
+		ConntrackMax int64   `json:"conntrack_max"`
+		RSS          int64   `json:"rss"`
+		CPUPermille  int64   `json:"cpu_permille"`
+		MemUsed      int64   `json:"mem_used"`
+		Load         float64 `json:"load"`
+	} `json:"latest"`
+}
+
+func statsCmd() *cobra.Command {
+	var since string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "stats", Short: "Traffic, drops, blocked sources and devices over time",
+		Long: `Sums up what the firewall did over a while (--since: 90m, 24h, 7d, 4w):
+traffic per zone, packets dropped, the busiest rules, the most connections
+at once, the sources blocked most and what they tried, and (with traffic
+per device on) the devices that used the most.
+
+  hc stats --since 7d
+  hc stats traffic --since 24h
+  hc stats top devices --since 30d
+  hc stats rules --unused`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s, err := get[statSummary]("/stats/summary", q("since", since, "limit", strconv.Itoa(limit)))
+			if err != nil {
+				return err
+			}
+			return output(s, func(w io.Writer) { printSummary(w, s) })
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d, 4w…")
+	cmd.Flags().IntVar(&limit, "limit", 5, "entries per top list")
+	cmd.AddCommand(statsStatusCmd(), statsTrafficCmd(), statsTopCmd(), statsRulesCmd(), statsConnectionsCmd())
+	return cmd
+}
+
+func printSummary(w io.Writer, s statSummary) {
+	fmt.Fprintf(w, "Since %s (%s)\n\n", s.From.Local().Format("2006-01-02 15:04"), s.Since)
+	if len(s.Zones) > 0 {
+		fmt.Fprintln(w, "ZONE\tRECEIVED\tSENT")
+		for _, t := range s.Zones {
+			fmt.Fprintf(w, "%s\t%s\t%s\n", t.Name, humanBytes(t.Received), humanBytes(t.Sent))
+		}
+		fmt.Fprintln(w)
+	}
+	var chains []string
+	for _, e := range s.DropsByChain {
+		chains = append(chains, fmt.Sprintf("%s %s", e.Key, humanCount(uint64(e.Count))))
+	}
+	drops := humanCount(uint64(s.Drops))
+	if len(chains) > 0 {
+		drops += " (" + strings.Join(chains, ", ") + ")"
+	}
+	fmt.Fprintf(w, "Dropped:\t%s packets\n", drops)
+	if s.ConntrackMax > 0 {
+		fmt.Fprintf(w, "Connections:\tat most %d at once (of %d)\n", s.PeakConnections, s.ConntrackMax)
+	}
+	list := func(title string, es []statEntry, unit func(int64) string) {
+		if len(es) == 0 {
+			return
+		}
+		fmt.Fprintf(w, "\n%s\n", title)
+		for _, e := range es {
+			name := e.Key
+			if e.Name != "" {
+				name = e.Name
+			}
+			fmt.Fprintf(w, "  %s\t%s\n", name, unit(e.Count))
+		}
+	}
+	packets := func(n int64) string { return humanCount(uint64(n)) + " packets" }
+	list("Busiest rules:", s.Rules, packets)
+	list("Blocked most (logged drops):", s.BlockedSources, packets)
+	list("What they tried:", s.BlockedPorts, packets)
+	list("Devices, sent:", s.DevicesUp, humanBytes)
+	list("Devices, received:", s.DevicesDown, humanBytes)
+}
+
+func statsStatusCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "status", Short: "Whether statistics are collected, since when, and the database", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s, err := get[statStatus]("/stats/status", nil)
+			if err != nil {
+				return err
+			}
+			return output(s, func(w io.Writer) {
+				if !s.Enabled {
+					fmt.Fprintln(w, "Statistics are off (stats.enabled in housecarl.yaml).")
+					return
+				}
+				fmt.Fprintf(w, "Collecting:\tyes\n")
+				if s.Database != "" {
+					fmt.Fprintf(w, "Database:\t%s (%s)\n", s.Database, humanBytes(s.Size))
+				} else {
+					fmt.Fprintf(w, "Database size:\t%s\n", humanBytes(s.Size))
+				}
+				if s.FirstData != nil {
+					fmt.Fprintf(w, "Since:\t%s\n", s.FirstData.Local().Format("2006-01-02 15:04"))
+				}
+				if s.LastSample != nil {
+					fmt.Fprintf(w, "Last sample:\t%s\n", s.LastSample.Local().Format("15:04:05"))
+				}
+				if s.SampleError != "" {
+					fmt.Fprintf(w, "Error:\t%s\n", s.SampleError)
+				}
+				fmt.Fprintf(w, "Kept:\tper minute %s, hourly %s, daily %s\n", s.Retention.Minute, s.Retention.Hour, s.Retention.Day)
+				fmt.Fprintf(w, "Per device:\t%s\n", map[bool]string{true: "on", false: "off (hc settings set --stats-per-device on)"}[s.PerDevice])
+				if l := s.Latest; l != nil {
+					if l.ConntrackMax > 0 {
+						fmt.Fprintf(w, "Connections now:\t%d of %d\n", l.Conntrack, l.ConntrackMax)
+					}
+					fmt.Fprintf(w, "housecarld:\t%s, CPU %.1f%%\n", humanBytes(l.RSS), float64(l.CPUPermille)/10)
+					fmt.Fprintf(w, "Host:\t%s in use, load %.2f\n", humanBytes(l.MemUsed), l.Load)
+				}
+			})
+		},
+	}
+}
+
+func statsTrafficCmd() *cobra.Command {
+	var since, step string
+	var ifaces bool
+	cmd := &cobra.Command{
+		Use: "traffic", Short: "Bytes received from and sent to each zone (or interface) over time", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			rx, tx := "zrx", "ztx"
+			if ifaces {
+				rx, tx = "rx", "tx"
+			}
+			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", rx+":*,"+tx+":*"))
+			if err != nil {
+				return err
+			}
+			return output(tl, func(w io.Writer) { printTimeline(w, tl, rx, tx) })
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
+	cmd.Flags().BoolVar(&ifaces, "interfaces", false, "per interface instead of per zone")
+	return cmd
+}
+
+// printTimeline writes a row per bucket: each name's received and sent.
+func printTimeline(w io.Writer, tl statTimeline, rx, tx string) {
+	var names []string
+	vals := map[string][]int64{}
+	for _, s := range tl.Series {
+		vals[s.Key] = s.Values
+		if n, ok := strings.CutPrefix(s.Key, rx+":"); ok {
+			names = append(names, n)
+		}
+	}
+	for _, s := range tl.Series {
+		if n, ok := strings.CutPrefix(s.Key, tx+":"); ok && vals[rx+":"+n] == nil {
+			names = append(names, n)
+		}
+	}
+	if len(names) == 0 {
+		fmt.Fprintln(w, "no traffic recorded yet")
+		return
+	}
+	fmt.Fprint(w, "TIME")
+	for _, n := range names {
+		fmt.Fprintf(w, "\t%s ↓\t%s ↑", n, n)
+	}
+	fmt.Fprintln(w)
+	layout := "01-02 15:04"
+	if tl.Step == "1d" {
+		layout = "2006-01-02"
+	}
+	at := func(k string, i int) int64 {
+		if v := vals[k]; i < len(v) {
+			return v[i]
+		}
+		return 0
+	}
+	for i, t := range tl.Times {
+		fmt.Fprint(w, time.Unix(t, 0).Local().Format(layout))
+		for _, n := range names {
+			fmt.Fprintf(w, "\t%s\t%s", humanBytes(at(rx+":"+n, i)), humanBytes(at(tx+":"+n, i)))
+		}
+		fmt.Fprintln(w)
+	}
+}
+
+func statsTopCmd() *cobra.Command {
+	var since string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "top devices|blocked|tried|quota RULE-ID", Short: "The largest: devices by traffic, sources blocked, what they tried, a quota's users",
+		Long: `Top lists over the last while:
+
+  devices   inside addresses by traffic to and from external zones (needs
+            hc settings set --stats-per-device on)
+  blocked   sources of logged drops (the policies' drops with 'log drops'
+            on, and rules that log)
+  tried     what those drops tried: tcp/22, udp/53, icmp/echo-request
+  quota ID  who used a per-address quota of rule ID`,
+		Args: cobra.RangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			get1 := func(kind string) ([]statEntry, error) {
+				return get[[]statEntry]("/stats/top", q("kind", kind, "since", since, "limit", strconv.Itoa(limit)))
+			}
+			switch args[0] {
+			case "devices":
+				up, err := get1("device_up")
+				if err != nil {
+					return err
+				}
+				down, err := get1("device_down")
+				if err != nil {
+					return err
+				}
+				out := map[string][2]int64{}
+				var order []string
+				for _, l := range [][]statEntry{down, up} {
+					for _, e := range l {
+						if _, ok := out[e.Key]; !ok {
+							order = append(order, e.Key)
+						}
+					}
+				}
+				for _, e := range down {
+					v := out[e.Key]
+					v[0] = e.Count
+					out[e.Key] = v
+				}
+				for _, e := range up {
+					v := out[e.Key]
+					v[1] = e.Count
+					out[e.Key] = v
+				}
+				return output(map[string]any{"up": up, "down": down}, func(w io.Writer) {
+					fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT")
+					for _, k := range order {
+						fmt.Fprintf(w, "%s\t%s\t%s\n", k, humanBytes(out[k][0]), humanBytes(out[k][1]))
+					}
+				})
+			case "blocked", "tried":
+				kind := map[string]string{"blocked": "blocked_src", "tried": "blocked_port"}[args[0]]
+				es, err := get1(kind)
+				if err != nil {
+					return err
+				}
+				return output(es, func(w io.Writer) {
+					fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS")
+					for _, e := range es {
+						fmt.Fprintf(w, "%s\t%d\n", e.Key, e.Count)
+					}
+				})
+			case "quota":
+				if len(args) != 2 {
+					return fmt.Errorf("give the rule's ID: hc stats top quota 12")
+				}
+				es, err := get1("quota_r" + args[1])
+				if err != nil {
+					return err
+				}
+				return output(es, func(w io.Writer) {
+					fmt.Fprintln(w, "ADDRESS\tUSED")
+					for _, e := range es {
+						fmt.Fprintf(w, "%s\t%s\n", e.Key, humanBytes(e.Count))
+					}
+				})
+			}
+			return fmt.Errorf("unknown list %q: devices, blocked, tried or quota", args[0])
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().IntVar(&limit, "limit", 20, "at most this many")
+	return cmd
+}
+
+func statsRulesCmd() *cobra.Command {
+	var since string
+	var unused bool
+	cmd := &cobra.Command{
+		Use: "rules", Short: "Packets each rule, built-in and policy counted; --unused: rules that matched nothing", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			es, err := get[[]statEntry]("/stats/hits", q("since", since))
+			if err != nil {
+				return err
+			}
+			if unused {
+				var keep []statEntry
+				for _, e := range es {
+					if e.Count == 0 && strings.HasPrefix(e.Key, "r") {
+						keep = append(keep, e)
+					}
+				}
+				es = keep
+			}
+			return output(es, func(w io.Writer) {
+				if unused && len(es) == 0 {
+					fmt.Fprintf(w, "every rule matched something in the last %s\n", since)
+					return
+				}
+				fmt.Fprintln(w, "WHAT\tPACKETS")
+				for _, e := range es {
+					fmt.Fprintf(w, "%s\t%d\n", orDash(e.Name), e.Count)
+				}
+			})
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "30d", "the last 24h, 7d, 30d…")
+	cmd.Flags().BoolVar(&unused, "unused", false, "only the rules that matched nothing")
+	return cmd
+}
+
+func statsConnectionsCmd() *cobra.Command {
+	var since, step string
+	cmd := &cobra.Command{
+		Use: "connections", Short: "How many connections the firewall tracked over time, against the limit", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", "g:conntrack,g:conntrack_max"))
+			if err != nil {
+				return err
+			}
+			return output(tl, func(w io.Writer) {
+				fmt.Fprintln(w, "TIME\tCONNECTIONS\tLIMIT")
+				for i, t := range tl.Times {
+					n, m := tl.Series[0].Values[i], tl.Series[1].Values[i]
+					if n < 0 {
+						continue // no sample
+					}
+					fmt.Fprintf(w, "%s\t%d\t%d\n", time.Unix(t, 0).Local().Format("01-02 15:04"), n, m)
+				}
+			})
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
+	return cmd
+}

cmd/housecarld/main.go

@@ -35,6 +35,7 @@ import (
 	"housecarl/internal/nft"
 	"housecarl/internal/render"
 	"housecarl/internal/server"
+	"housecarl/internal/stats"
 	"housecarl/internal/store"
 	"housecarl/internal/trace"
 	"housecarl/internal/validate"
@@ -198,6 +199,7 @@ func serveCmd() *cobra.Command {
 			go nflog.Listen(ctx, render.LogGroup, a.log, srv.LogPacket)
 			go srv.RunQuotas(ctx)
 			srv.SetConntrack(conntrack.Client{})
+			statsDone := startStats(ctx, a, fw, srv)
 			// A trace left behind by a crash.
 			fw.DeleteTable(ctx, "inet", trace.Table)
 			hup := make(chan os.Signal, 1)
@@ -212,12 +214,45 @@ func serveCmd() *cobra.Command {
 			err = srv.Serve(ctx, func() { sdNotify("READY=1") })
 			sdNotify("STOPPING=1")
 			srv.SaveQuotaUsage(context.Background())
+			<-statsDone
 			a.log.Info("stopped")
 			return err
 		},
 	}
 }
 
+// startStats opens the statistics database and runs the collector until
+// ctx ends; the channel closes once it wrote its last figures.
+func startStats(ctx context.Context, a *app, fw *nft.Firewall, srv *server.Server) <-chan struct{} {
+	done := make(chan struct{})
+	c := a.cfg.Stats
+	if !c.Enabled {
+		close(done)
+		return done
+	}
+	st, err := stats.OpenStore(c.Database)
+	if err != nil {
+		a.log.Error("statistics are off: opening their database failed (deleting it is safe)", "err", err)
+		close(done)
+		return done
+	}
+	opts := stats.Options{TopN: c.TopN, RetainMinute: time.Duration(c.Retention.Minute), RetainHour: time.Duration(c.Retention.Hour), RetainDay: time.Duration(c.Retention.Day)}
+	// Tests sample more often than every minute.
+	if v := os.Getenv("HOUSECARL_STATS_INTERVAL"); v != "" {
+		opts.Interval, _ = time.ParseDuration(v)
+	}
+	src := stats.Sources{Table: fw.Table}
+	srv.StatsSources(&src)
+	col := stats.NewCollector(st, opts, src, a.log)
+	srv.SetStats(col)
+	go func() {
+		defer close(done)
+		defer st.Close()
+		col.Run(ctx)
+	}()
+	return done
+}
+
 func sdNotify(state string) {
 	sock := os.Getenv("NOTIFY_SOCKET")
 	if sock == "" {