Statistics: traffic, drops, devices and connections over time

internal/stats samples every minute what the kernel already counts: the
interfaces' counters (per zone too), Housecarl's table (hits per rule,
built-in and policy; drops by chain; quotas; per-device sets), the
connection tracking table, and housecarld's and the host's memory, CPU
and load. Logged drops feed top lists of sources and what they tried.
Aggregates go to a database of their own (stats.db; per minute 7 days,
per hour 90 days, per day 2 years), with the store and queries from
Jarl's.

Traffic per inside device is a setting (stats_per_device): counting sets
stats_up/down_v4/v6 and rules at the top of the forward chain, IPv6 per
/64, carried across reloads. Set names starting with meter_, conn_,
quota_ or stats_ are reserved (set_name_reserved).

GET /stats/{status,summary,timeline,top,hits,heatmap}, hc stats, a
Statistics page with Jarl's charts, two charts on the dashboard, and a
conntrack_full warning. e2e: 188 checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/h_stats.go

@@ -0,0 +1,266 @@
+package server
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"housecarl/internal/api"
+	"housecarl/internal/auth"
+	"housecarl/internal/model"
+	"housecarl/internal/stats"
+	"housecarl/internal/validate"
+)
+
+// Statistics: the firewall's figures over time (internal/stats).
+
+// StatsSources fills in what the statistics collector needs from the
+// server: the interfaces' zones, and which counters count drops.
+func (s *Server) StatsSources(src *stats.Sources) {
+	src.Zones = s.interfaceZones
+	src.Classify = s.classifyMark
+}
+
+// SetStats gives the server the statistics collector (nil: statistics
+// are off).
+func (s *Server) SetStats(c *stats.Collector) { s.stats = c }
+
+// interfaceZones maps the machine's interfaces to their zones.
+func (s *Server) interfaceZones() map[string]string {
+	out := map[string]string{}
+	for _, i := range s.hostInterfaces(s.eng.State()) {
+		if i.Zone != "" {
+			out[i.Name] = i.Zone
+		}
+	}
+	return out
+}
+
+// classifyMark says which chain a counter is in and whether what it
+// counts is dropped: rules that drop or reject, the drop policies, and
+// the built-ins that drop (antispoofing, invalid packets).
+func (s *Server) classifyMark(mark string) (string, bool) {
+	st := s.eng.State()
+	switch {
+	case strings.HasPrefix(mark, "p-"):
+		chain := strings.TrimPrefix(mark, "p-")
+		policy := map[string]string{"input": st.Settings.InputPolicy, "forward": st.Settings.ForwardPolicy, "output": st.Settings.OutputPolicy}[chain]
+		return chain, policy == model.PolicyDrop
+	case strings.HasPrefix(mark, "b-antispoof"):
+		return model.ChainEarly, true
+	case mark == "b-invalid":
+		return "invalid", true
+	case strings.HasPrefix(mark, "r"):
+		num := strings.TrimLeft(mark[1:], "0123456789")
+		id, _ := strconv.ParseInt(strings.TrimSuffix(mark[1:], num), 10, 64)
+		if o, ok := st.ByID(model.KindRule, id).(*model.Rule); ok && o != nil {
+			return o.Chain, o.Action == model.ActionDrop || o.Action == model.ActionReject
+		}
+	}
+	return "", false
+}
+
+// markName names a counter's object: "rule input #12 (ssh)", "built-in:
+// established", "input policy".
+func markName(st *model.State, mark string) string {
+	if c, ok := strings.CutPrefix(mark, "p-"); ok {
+		return c + " policy"
+	}
+	if l := markLabel(st, api.TraceStep{Kind: "rule", Mark: mark}); l != "" {
+		return l
+	}
+	if len(mark) < 2 {
+		return mark
+	}
+	// An object deleted since it counted.
+	kind := map[byte]string{'r': "rule", 'f': "port forward", 's': "source NAT", 'd': "redirect"}[mark[0]]
+	if num := strings.TrimLeft(mark[1:], "0123456789"); kind != "" && len(num) < len(mark)-1 {
+		return kind + " #" + strings.TrimSuffix(mark[1:], num) + " (deleted)"
+	}
+	return mark
+}
+
+func (s *Server) statsCollector() (*stats.Collector, error) {
+	if s.stats == nil {
+		return nil, &httpError{http.StatusServiceUnavailable, "stats_off", "statistics are off: set stats.enabled in housecarl.yaml and restart housecarld"}
+	}
+	return s.stats, nil
+}
+
+func (s *Server) statsQuerier(p *auth.Principal) (*stats.Querier, error) {
+	if err := require(p, auth.ResLog, false); err != nil {
+		return nil, err
+	}
+	c, err := s.statsCollector()
+	if err != nil {
+		return nil, err
+	}
+	q := stats.NewQuerier(c)
+	q.Now = s.nowFn
+	return q, nil
+}
+
+func since(r *http.Request, def string) (string, error) {
+	v := r.URL.Query().Get("since")
+	if v == "" {
+		v = def
+	}
+	if _, err := stats.ParseSince(v); err != nil {
+		return "", badRequest("%v", err)
+	}
+	return v, nil
+}
+
+// StatsStatus is the collector's health, with the setting that counts
+// traffic per device.
+type StatsStatus struct {
+	stats.Status
+	Latest *stats.Latest `json:"latest,omitempty"`
+}
+
+func (s *Server) statsStatus(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := require(p, auth.ResLog, false); err != nil {
+		return err
+	}
+	out := StatsStatus{}
+	if s.stats != nil {
+		out.Status = s.stats.Status()
+		out.Latest = s.stats.Latest()
+		if !p.IsAdmin() {
+			out.Database = ""
+		}
+	}
+	out.PerDevice = s.eng.State().Settings.StatsPerDevice
+	writeJSON(w, http.StatusOK, out)
+	return nil
+}
+
+func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(p)
+	if err != nil {
+		return err
+	}
+	v, err := since(r, "24h")
+	if err != nil {
+		return err
+	}
+	d, _ := stats.ParseSince(v)
+	sum, err := q.Summary(r.Context(), d, queryInt(r, "limit", 10))
+	if err != nil {
+		return err
+	}
+	st := s.eng.State()
+	for i := range sum.Rules {
+		sum.Rules[i].Name = markName(st, sum.Rules[i].Key)
+	}
+	writeJSON(w, http.StatusOK, sum)
+	return nil
+}
+
+func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(p)
+	if err != nil {
+		return err
+	}
+	v, err := since(r, "24h")
+	if err != nil {
+		return err
+	}
+	d, _ := stats.ParseSince(v)
+	var keys []string
+	for _, k := range strings.Split(r.URL.Query().Get("keys"), ",") {
+		if k = strings.TrimSpace(k); k != "" {
+			keys = append(keys, k)
+		}
+	}
+	if len(keys) == 0 {
+		return badRequest("say which figures: keys=zrx:*,ztx:* (docs/stats.md)")
+	}
+	tl, err := q.Timeline(r.Context(), d, r.URL.Query().Get("step"), keys)
+	if err != nil {
+		return badRequest("%v", err)
+	}
+	writeJSON(w, http.StatusOK, tl)
+	return nil
+}
+
+func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(p)
+	if err != nil {
+		return err
+	}
+	v, err := since(r, "24h")
+	if err != nil {
+		return err
+	}
+	d, _ := stats.ParseSince(v)
+	es, err := q.Top(r.Context(), r.URL.Query().Get("kind"), d, queryInt(r, "limit", 20))
+	if err != nil {
+		return badRequest("%v", err)
+	}
+	writeJSON(w, http.StatusOK, es)
+	return nil
+}
+
+// statsHits are the packets each rule, built-in and policy counted over
+// the range, named; rules that counted none are listed too, with 0.
+func (s *Server) statsHits(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(p)
+	if err != nil {
+		return err
+	}
+	v, err := since(r, "30d")
+	if err != nil {
+		return err
+	}
+	d, _ := stats.ParseSince(v)
+	es, err := q.Sums(r.Context(), "hits:", d)
+	if err != nil {
+		return err
+	}
+	st := s.eng.State()
+	seen := map[string]bool{}
+	for i := range es {
+		es[i].Name = markName(st, es[i].Key)
+		seen[es[i].Key] = true
+	}
+	for _, rule := range st.Rules {
+		if mark := "r" + strconv.FormatInt(rule.ID, 10); !seen[mark] && !rule.Disabled {
+			es = append(es, stats.Entry{Key: mark, Name: "rule " + rule.Label()})
+		}
+	}
+	writeJSON(w, http.StatusOK, es)
+	return nil
+}
+
+func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	q, err := s.statsQuerier(p)
+	if err != nil {
+		return err
+	}
+	v, err := since(r, "28d")
+	if err != nil {
+		return err
+	}
+	d, _ := stats.ParseSince(v)
+	hm, err := q.Heatmap(r.Context(), r.URL.Query().Get("key"), d)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, hm)
+	return nil
+}
+
+// statsProblems warns when the connection tracking table is nearly full:
+// new connections are dropped once it is.
+func (s *Server) statsProblems() []validate.Problem {
+	if s.stats == nil {
+		return nil
+	}
+	l := s.stats.Latest()
+	if l == nil || l.ConntrackMax == 0 || l.Conntrack*100 < l.ConntrackMax*80 {
+		return nil
+	}
+	return []validate.Problem{{Severity: validate.Warning, Code: "conntrack_full", Object: "connections",
+		Message: "the connection tracking table is " + strconv.FormatInt(l.Conntrack*100/l.ConntrackMax, 10) + "% full (" + strconv.FormatInt(l.Conntrack, 10) + " of " + strconv.FormatInt(l.ConntrackMax, 10) + "); once it is, new connections are dropped: raise net.netfilter.nf_conntrack_max, or find what opens so many (Connections)"}}
+}

internal/server/h_status.go

@@ -48,6 +48,7 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa
 		out.OtherTables = otherTables(data)
 	}
 	out.Problems = append(out.Problems, s.hostProblems(r.Context(), out.OtherTables)...)
+	out.Problems = append(out.Problems, s.statsProblems()...)
 	writeJSON(w, http.StatusOK, out)
 	return nil
 }
@@ -296,7 +297,12 @@ func (l *logRing) Latest(n int, after time.Time, keep func(api.LogEntry) bool) [
 
 // LogPacket records a packet the firewall logged (internal/nflog calls
 // it).
-func (s *Server) LogPacket(e api.LogEntry) { s.logs.Add(e) }
+func (s *Server) LogPacket(e api.LogEntry) {
+	s.logs.Add(e)
+	if s.stats != nil {
+		s.stats.Packet(e)
+	}
+}
 
 func (s *Server) listLog(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
 	if err := require(p, auth.ResLog, false); err != nil {

internal/server/routes.go

@@ -64,6 +64,12 @@ func (s *Server) routes() {
 	m.HandleFunc("POST "+p+"/{kind}/{ref}/move", s.authed(s.moveObject))
 	m.HandleFunc("GET "+p+"/ruleset", s.authed(s.ruleset))
 	m.HandleFunc("GET "+p+"/matrix", s.authed(s.zoneMatrix))
+	m.HandleFunc("GET "+p+"/stats/status", s.authed(s.statsStatus))
+	m.HandleFunc("GET "+p+"/stats/summary", s.authed(s.statsSummary))
+	m.HandleFunc("GET "+p+"/stats/timeline", s.authed(s.statsTimeline))
+	m.HandleFunc("GET "+p+"/stats/top", s.authed(s.statsTop))
+	m.HandleFunc("GET "+p+"/stats/hits", s.authed(s.statsHits))
+	m.HandleFunc("GET "+p+"/stats/heatmap", s.authed(s.statsHeatmap))
 	m.HandleFunc("GET "+p+"/history", s.authed(s.listHistory))
 	m.HandleFunc("GET "+p+"/history/{id}", s.authed(s.getHistory))
 	m.HandleFunc("POST "+p+"/history/{id}/revert", s.authed(s.revertHistory))

internal/server/server.go

@@ -24,6 +24,7 @@ import (
 	"housecarl/internal/model"
 	"housecarl/internal/nft"
 	"housecarl/internal/render"
+	"housecarl/internal/stats"
 	"housecarl/internal/store"
 	"housecarl/internal/version"
 )
@@ -97,6 +98,8 @@ type Server struct {
 	conns Conntrack
 	// tracing is the packet trace running (or the last one).
 	tracing tracing
+	// stats collects statistics (nil: off).
+	stats *stats.Collector
 	// csMu orders changes to changesets (staging, applying, discarding).
 	csMu  sync.Mutex
 	nowFn func() time.Time

internal/server/server_test.go

@@ -23,6 +23,7 @@ import (
 	"housecarl/internal/model"
 	"housecarl/internal/nft"
 	"housecarl/internal/render"
+	"housecarl/internal/stats"
 	"housecarl/internal/store"
 )
 
@@ -994,3 +995,67 @@ func TestTrace(t *testing.T) {
 		t.Errorf("tracing everything: %d", code)
 	}
 }
+
+// TestStatsEndpoints: statistics off, then on: the summary's top lists,
+// hits per rule (named, with those that counted none), and which
+// counters count drops.
+func TestStatsEndpoints(t *testing.T) {
+	e := newEnv(t)
+	if code := e.call(e.token, "GET", "/stats/summary", nil, nil); code != http.StatusServiceUnavailable {
+		t.Errorf("statistics off: %d", code)
+	}
+	var st StatsStatus
+	e.call(e.token, "GET", "/stats/status", nil, &st)
+	if st.Enabled {
+		t.Error("enabled without a collector")
+	}
+	e.call(e.token, "POST", "/zones", model.Zone{Name: "wan", Interfaces: []string{"wan"}, External: true}, nil)
+	e.call(e.token, "POST", "/rules", model.Rule{Chain: "input", Match: model.Match{Sources: []string{"198.51.100.0/24"}}, Action: "drop", Comment: "bad net"}, nil)
+	rule := e.eng.State().Rules[0]
+
+	ss, err := stats.OpenStore(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close()
+	src := stats.Sources{Table: e.fw.Table, SysNet: t.TempDir(), Proc: t.TempDir()}
+	e.srv.StatsSources(&src)
+	c := stats.NewCollector(ss, stats.Options{RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour}, src, nil)
+	e.srv.SetStats(c)
+	c.Start(context.Background())
+	e.srv.LogPacket(api.LogEntry{Time: time.Now(), Source: "198.51.100.7", Protocol: "tcp", DestPort: 22, Verdict: "drop"})
+	c.Sample(context.Background())
+
+	var sum stats.Summary
+	if code := e.call(e.token, "GET", "/stats/summary?since=1h", nil, &sum); code != http.StatusOK {
+		t.Fatalf("summary: %d", code)
+	}
+	if len(sum.BlockedSources) != 1 || sum.BlockedSources[0].Key != "198.51.100.7" || len(sum.BlockedPorts) != 1 || sum.BlockedPorts[0].Key != "tcp/22" {
+		t.Errorf("summary: %+v", sum)
+	}
+	var hits []stats.Entry
+	e.call(e.token, "GET", "/stats/hits", nil, &hits)
+	if len(hits) != 1 || hits[0].Key != "r"+itoa(rule.ID) || hits[0].Count != 0 || !strings.Contains(hits[0].Name, "bad net") {
+		t.Errorf("hits: %+v", hits)
+	}
+	var top []stats.Entry
+	if code := e.call(e.token, "GET", "/stats/top?kind=blocked_port&since=7d", nil, &top); code != http.StatusOK || len(top) != 1 {
+		t.Errorf("top: %d %+v", code, top)
+	}
+	if code := e.call(e.token, "GET", "/stats/top?kind=nonsense", nil, nil); code != http.StatusBadRequest {
+		t.Errorf("an unknown list: %d", code)
+	}
+	if code := e.call(e.token, "GET", "/stats/timeline?keys=zrx:*,drops&since=2h", nil, nil); code != http.StatusOK {
+		t.Errorf("timeline: %d", code)
+	}
+	e.call(e.token, "GET", "/stats/status", nil, &st)
+	if !st.Enabled || st.LastSample == nil {
+		t.Errorf("status: %+v", st)
+	}
+	for mark, want := range map[string]string{"r" + itoa(rule.ID): "input true", "p-input": "input true", "p-output": "output false", "b-antispoof6": "early true", "b-established": " false"} {
+		chain, drop := e.srv.classifyMark(mark)
+		if got := fmt.Sprint(chain, " ", drop); got != want {
+			t.Errorf("%s: %q, want %q", mark, got, want)
+		}
+	}
+}