From cf30715c5ad767a94f2772ff4bec1e119d7872fc Mon Sep 17 00:00:00 2001 From: abrown38 Date: Mon, 19 Oct 2026 22:27:48 +0100 Subject: [PATCH] Statistics: traffic, drops, devices and connections over time internal/stats samples every minute what the kernel already counts: the interfaces' counters (per zone too), Housecarl's table (hits per rule, built-in and policy; drops by chain; quotas; per-device sets), the connection tracking table, and housecarld's and the host's memory, CPU and load. Logged drops feed top lists of sources and what they tried. Aggregates go to a database of their own (stats.db; per minute 7 days, per hour 90 days, per day 2 years), with the store and queries from Jarl's. Traffic per inside device is a setting (stats_per_device): counting sets stats_up/down_v4/v6 and rules at the top of the forward chain, IPv6 per /64, carried across reloads. Set names starting with meter_, conn_, quota_ or stats_ are reserved (set_name_reserved). GET /stats/{status,summary,timeline,top,hits,heatmap}, hc stats, a Statistics page with Jarl's charts, two charts on the dashboard, and a conntrack_full warning. e2e: 188 checks. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 + README.md | 3 +- cmd/hc/firewall.go | 14 +- cmd/hc/main.go | 2 +- cmd/hc/stats.go | 401 +++++++++++++ cmd/housecarld/main.go | 35 ++ docs/api.md | 9 +- docs/cli.md | 3 +- docs/configuration.md | 7 +- docs/stats.md | 140 +++++ docs/validation.md | 4 + internal/config/config.go | 36 +- internal/model/settings.go | 4 + internal/nft/nft.go | 23 +- internal/nft/nft_test.go | 49 ++ internal/render/render.go | 58 ++ internal/render/render_test.go | 32 +- internal/server/h_stats.go | 266 +++++++++ internal/server/h_status.go | 8 +- internal/server/routes.go | 6 + internal/server/server.go | 3 + internal/server/server_test.go | 65 +++ internal/stats/collector.go | 546 ++++++++++++++++++ internal/stats/collector_test.go | 238 ++++++++ internal/stats/query.go | 480 +++++++++++++++ internal/stats/sources.go | 136 +++++ internal/stats/store.go | 304 ++++++++++ internal/validate/validate.go | 11 + internal/validate/validate_test.go | 9 + internal/web/static/assets/app.js | 3 + internal/web/static/assets/charts.js | 351 +++++++++++ internal/web/static/assets/pages/dashboard.js | 24 +- internal/web/static/assets/pages/settings.js | 6 +- internal/web/static/assets/pages/stats.js | 214 +++++++ internal/web/static/assets/pages/trace.js | 4 +- internal/web/static/assets/style.css | 133 +++++ scripts/e2e.sh | 30 +- 37 files changed, 3652 insertions(+), 18 deletions(-) create mode 100644 cmd/hc/stats.go create mode 100644 docs/stats.md create mode 100644 internal/server/h_stats.go create mode 100644 internal/stats/collector.go create mode 100644 internal/stats/collector_test.go create mode 100644 internal/stats/query.go create mode 100644 internal/stats/sources.go create mode 100644 internal/stats/store.go create mode 100644 internal/web/static/assets/charts.js create mode 100644 internal/web/static/assets/pages/stats.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eaf852..f8e3985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,19 @@ the way Jarl and Steward are. - **Warnings about the machine:** while `nftables.service` is enabled (it loads `/etc/nftables.conf` at boot, and flushes every table when stopped), and while the table an import came from still applies. +- **Statistics** (Traffic → Statistics, `hc stats`, `/api/v1/stats/...`): + traffic per interface and zone, drops by chain and when they happen, + hits per rule and the rules that matched nothing, the sources blocked + most and what they tried, connections against the kernel's limit, each + quota's use, and housecarld's and the host's memory and load, sampled + every minute into a database of their own (`stats.db`, kept 7 days per + minute, 90 days per hour, 2 years per day). On by default; traffic per + inside device is a setting (`--stats-per-device`) that adds counters + to the forward chain. Two charts on the dashboard; a warning when the + connection tracking table is nearly full (docs/stats.md). +- Set names starting with `meter_`, `conn_`, `quota_` or `stats_` are + refused (`set_name_reserved`): Housecarl's own sets use them, and a set + of yours named so could clash with them. - **The zone matrix** (Matrix, `hc matrix`): what the rules do to new connections between zones and to and from the firewall, worked out from the rules in order, with each pair's exceptions; rules about diff --git a/README.md b/README.md index 3611bbd..345cd5a 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ table of its own. rules' order, and keeps what it can't translate as raw rules ([docs/import.md](docs/import.md)). * **Easy to watch.** A zone matrix of what is allowed between zones, - hits per rule, a live log of what was dropped and + statistics over time (traffic per zone and device, drops, the sources + blocked most, connections; [docs/stats.md](docs/stats.md)), hits per rule, a live log of what was dropped and why, the connections the firewall tracks (and closing them), a packet tracer that names the rule that dropped something, in any table, and the history of every change, field by field, with undo. diff --git a/cmd/hc/firewall.go b/cmd/hc/firewall.go index 7982141..b6fc972 100644 --- a/cmd/hc/firewall.go +++ b/cmd/hc/firewall.go @@ -125,6 +125,7 @@ func settingsCmd() *cobra.Command { fmt.Fprintf(w, "Antispoof IPv6:\t%s\n", list(s.AntispoofV6)) fmt.Fprintf(w, "Flowtable:\t%s\n", list(s.FlowtableDevices)) fmt.Fprintf(w, "Probation:\t%s\n", fmtSeconds(s.Probation())) + fmt.Fprintf(w, "Traffic per device:\t%s\n", map[bool]string{true: "counted (statistics)", false: "not counted"}[s.StatsPerDevice]) if s.ManualConfirm { fmt.Fprintln(w, "Confirmation:\tby a person ('hc confirm', the web UI's banner)") } else { @@ -133,7 +134,7 @@ func settingsCmd() *cobra.Command { }) }, }) - var input, forward, output, logDrops, probation, autoConf string + var input, forward, output, logDrops, probation, autoConf, perDevice string var builtins, own, flowtable, spoof4, spoof6 []string set := &cobra.Command{ Use: "set [flags]", Short: "Change settings (only the flags given)", Args: cobra.NoArgs, @@ -143,7 +144,8 @@ dhcpv6_client, antispoof, mss_clamp. hc settings set --builtin mss_clamp=on --own-prefix 2001:db8::/48 hc settings set --forward-policy drop --log-drops on - hc settings set --flowtable wan --flowtable eth1`, + hc settings set --flowtable wan --flowtable eth1 + hc settings set --stats-per-device on`, RunE: func(cmd *cobra.Command, args []string) error { fl := cmd.Flags() p := patch{} @@ -177,6 +179,13 @@ dhcpv6_client, antispoof, mss_clamp. p[map[string]string{"own-prefix": "own_prefixes", "flowtable": "flowtable_devices", "antispoof-v4": "antispoof_v4", "antispoof-v6": "antispoof_v6"}[flag]] = v } } + if fl.Changed("stats-per-device") { + on, err := onOff("stats-per-device", perDevice) + if err != nil { + return err + } + p["stats_per_device"] = on + } if fl.Changed("auto-confirm") { on, err := onOff("auto-confirm", autoConf) if err != nil { @@ -205,6 +214,7 @@ dhcpv6_client, antispoof, mss_clamp. f.StringArrayVar(&spoof4, "antispoof-v4", nil, "IPv4 sources dropped from the internet (replaces)") f.StringArrayVar(&spoof6, "antispoof-v6", nil, "IPv6 sources dropped from the internet (replaces)") f.StringVar(&probation, "probation", "", "how long a network change has to be confirmed (2m)") + f.StringVar(&perDevice, "stats-per-device", "", "count traffic per inside device, for the statistics' top devices: on or off") f.StringVar(&autoConf, "auto-confirm", "", "on: the client that made a change confirms it once it reaches Housecarl again over a new connection; off: a person confirms") cmd.AddCommand(set) return cmd diff --git a/cmd/hc/main.go b/cmd/hc/main.go index 218364b..33061cd 100644 --- a/cmd/hc/main.go +++ b/cmd/hc/main.go @@ -191,7 +191,7 @@ Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found, } } add("fw", statusCmd(), zoneCmd(), setCmd(), serviceCmd(), ruleCmd(), forwardCmd(), snatCmd(), redirectCmd(), settingsCmd(), - rulesetCmd(), countersCmd(), quotasCmd(), logCmd(), connectionsCmd(), traceCmd(), matrixCmd(), interfaceCmd()) + rulesetCmd(), countersCmd(), quotasCmd(), logCmd(), connectionsCmd(), traceCmd(), matrixCmd(), statsCmd(), interfaceCmd()) add("changes", changesetCmd(), confirmCmd(), rollbackCmd(), historyCmd(), importCmd(), takeoverCmd()) add("admin", whoamiCmd(), userCmd(), tokenCmd(), passwordCmd(), certCmd(), auditCmd(), configCmd()) return root diff --git a/cmd/hc/stats.go b/cmd/hc/stats.go new file mode 100644 index 0000000..4a13449 --- /dev/null +++ b/cmd/hc/stats.go @@ -0,0 +1,401 @@ +package main + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" +) + +// Statistics: the figures housecarld keeps over time (docs/stats.md). The +// types mirror internal/stats's JSON, so hc doesn't link the database. + +type statEntry struct { + Key string `json:"key"` + Count int64 `json:"count"` + Name string `json:"name,omitempty"` +} + +type statTraffic struct { + Name string `json:"name"` + Received int64 `json:"received"` + Sent int64 `json:"sent"` +} + +type statSummary struct { + Since string `json:"since"` + From time.Time `json:"from"` + Zones []statTraffic `json:"zones"` + Interfaces []statTraffic `json:"interfaces"` + Drops int64 `json:"drops"` + DropsByChain []statEntry `json:"drops_by_chain"` + Rules []statEntry `json:"rules"` + PeakConnections int64 `json:"peak_connections"` + ConntrackMax int64 `json:"conntrack_max"` + BlockedSources []statEntry `json:"blocked_sources"` + BlockedPorts []statEntry `json:"blocked_ports"` + DevicesUp []statEntry `json:"devices_up"` + DevicesDown []statEntry `json:"devices_down"` +} + +type statTimeline struct { + Step string `json:"step"` + Times []int64 `json:"times"` + Series []struct { + Key string `json:"key"` + Values []int64 `json:"values"` + } `json:"series"` +} + +type statStatus struct { + Enabled bool `json:"enabled"` + Database string `json:"database"` + Size int64 `json:"size"` + LastSample *time.Time `json:"last_sample"` + SampleError string `json:"sample_error"` + FirstData *time.Time `json:"first_data"` + PerDevice bool `json:"per_device"` + Retention struct { + Minute, Hour, Day string + } `json:"retention"` + Latest *struct { + Conntrack int64 `json:"conntrack"` + ConntrackMax int64 `json:"conntrack_max"` + RSS int64 `json:"rss"` + CPUPermille int64 `json:"cpu_permille"` + MemUsed int64 `json:"mem_used"` + Load float64 `json:"load"` + } `json:"latest"` +} + +func statsCmd() *cobra.Command { + var since string + var limit int + cmd := &cobra.Command{ + Use: "stats", Short: "Traffic, drops, blocked sources and devices over time", + Long: `Sums up what the firewall did over a while (--since: 90m, 24h, 7d, 4w): +traffic per zone, packets dropped, the busiest rules, the most connections +at once, the sources blocked most and what they tried, and (with traffic +per device on) the devices that used the most. + + hc stats --since 7d + hc stats traffic --since 24h + hc stats top devices --since 30d + hc stats rules --unused`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + s, err := get[statSummary]("/stats/summary", q("since", since, "limit", strconv.Itoa(limit))) + if err != nil { + return err + } + return output(s, func(w io.Writer) { printSummary(w, s) }) + }, + } + cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d, 4w…") + cmd.Flags().IntVar(&limit, "limit", 5, "entries per top list") + cmd.AddCommand(statsStatusCmd(), statsTrafficCmd(), statsTopCmd(), statsRulesCmd(), statsConnectionsCmd()) + return cmd +} + +func printSummary(w io.Writer, s statSummary) { + fmt.Fprintf(w, "Since %s (%s)\n\n", s.From.Local().Format("2006-01-02 15:04"), s.Since) + if len(s.Zones) > 0 { + fmt.Fprintln(w, "ZONE\tRECEIVED\tSENT") + for _, t := range s.Zones { + fmt.Fprintf(w, "%s\t%s\t%s\n", t.Name, humanBytes(t.Received), humanBytes(t.Sent)) + } + fmt.Fprintln(w) + } + var chains []string + for _, e := range s.DropsByChain { + chains = append(chains, fmt.Sprintf("%s %s", e.Key, humanCount(uint64(e.Count)))) + } + drops := humanCount(uint64(s.Drops)) + if len(chains) > 0 { + drops += " (" + strings.Join(chains, ", ") + ")" + } + fmt.Fprintf(w, "Dropped:\t%s packets\n", drops) + if s.ConntrackMax > 0 { + fmt.Fprintf(w, "Connections:\tat most %d at once (of %d)\n", s.PeakConnections, s.ConntrackMax) + } + list := func(title string, es []statEntry, unit func(int64) string) { + if len(es) == 0 { + return + } + fmt.Fprintf(w, "\n%s\n", title) + for _, e := range es { + name := e.Key + if e.Name != "" { + name = e.Name + } + fmt.Fprintf(w, " %s\t%s\n", name, unit(e.Count)) + } + } + packets := func(n int64) string { return humanCount(uint64(n)) + " packets" } + list("Busiest rules:", s.Rules, packets) + list("Blocked most (logged drops):", s.BlockedSources, packets) + list("What they tried:", s.BlockedPorts, packets) + list("Devices, sent:", s.DevicesUp, humanBytes) + list("Devices, received:", s.DevicesDown, humanBytes) +} + +func statsStatusCmd() *cobra.Command { + return &cobra.Command{ + Use: "status", Short: "Whether statistics are collected, since when, and the database", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + s, err := get[statStatus]("/stats/status", nil) + if err != nil { + return err + } + return output(s, func(w io.Writer) { + if !s.Enabled { + fmt.Fprintln(w, "Statistics are off (stats.enabled in housecarl.yaml).") + return + } + fmt.Fprintf(w, "Collecting:\tyes\n") + if s.Database != "" { + fmt.Fprintf(w, "Database:\t%s (%s)\n", s.Database, humanBytes(s.Size)) + } else { + fmt.Fprintf(w, "Database size:\t%s\n", humanBytes(s.Size)) + } + if s.FirstData != nil { + fmt.Fprintf(w, "Since:\t%s\n", s.FirstData.Local().Format("2006-01-02 15:04")) + } + if s.LastSample != nil { + fmt.Fprintf(w, "Last sample:\t%s\n", s.LastSample.Local().Format("15:04:05")) + } + if s.SampleError != "" { + fmt.Fprintf(w, "Error:\t%s\n", s.SampleError) + } + fmt.Fprintf(w, "Kept:\tper minute %s, hourly %s, daily %s\n", s.Retention.Minute, s.Retention.Hour, s.Retention.Day) + fmt.Fprintf(w, "Per device:\t%s\n", map[bool]string{true: "on", false: "off (hc settings set --stats-per-device on)"}[s.PerDevice]) + if l := s.Latest; l != nil { + if l.ConntrackMax > 0 { + fmt.Fprintf(w, "Connections now:\t%d of %d\n", l.Conntrack, l.ConntrackMax) + } + fmt.Fprintf(w, "housecarld:\t%s, CPU %.1f%%\n", humanBytes(l.RSS), float64(l.CPUPermille)/10) + fmt.Fprintf(w, "Host:\t%s in use, load %.2f\n", humanBytes(l.MemUsed), l.Load) + } + }) + }, + } +} + +func statsTrafficCmd() *cobra.Command { + var since, step string + var ifaces bool + cmd := &cobra.Command{ + Use: "traffic", Short: "Bytes received from and sent to each zone (or interface) over time", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + rx, tx := "zrx", "ztx" + if ifaces { + rx, tx = "rx", "tx" + } + tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", rx+":*,"+tx+":*")) + if err != nil { + return err + } + return output(tl, func(w io.Writer) { printTimeline(w, tl, rx, tx) }) + }, + } + cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…") + cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d") + cmd.Flags().BoolVar(&ifaces, "interfaces", false, "per interface instead of per zone") + return cmd +} + +// printTimeline writes a row per bucket: each name's received and sent. +func printTimeline(w io.Writer, tl statTimeline, rx, tx string) { + var names []string + vals := map[string][]int64{} + for _, s := range tl.Series { + vals[s.Key] = s.Values + if n, ok := strings.CutPrefix(s.Key, rx+":"); ok { + names = append(names, n) + } + } + for _, s := range tl.Series { + if n, ok := strings.CutPrefix(s.Key, tx+":"); ok && vals[rx+":"+n] == nil { + names = append(names, n) + } + } + if len(names) == 0 { + fmt.Fprintln(w, "no traffic recorded yet") + return + } + fmt.Fprint(w, "TIME") + for _, n := range names { + fmt.Fprintf(w, "\t%s ↓\t%s ↑", n, n) + } + fmt.Fprintln(w) + layout := "01-02 15:04" + if tl.Step == "1d" { + layout = "2006-01-02" + } + at := func(k string, i int) int64 { + if v := vals[k]; i < len(v) { + return v[i] + } + return 0 + } + for i, t := range tl.Times { + fmt.Fprint(w, time.Unix(t, 0).Local().Format(layout)) + for _, n := range names { + fmt.Fprintf(w, "\t%s\t%s", humanBytes(at(rx+":"+n, i)), humanBytes(at(tx+":"+n, i))) + } + fmt.Fprintln(w) + } +} + +func statsTopCmd() *cobra.Command { + var since string + var limit int + cmd := &cobra.Command{ + Use: "top devices|blocked|tried|quota RULE-ID", Short: "The largest: devices by traffic, sources blocked, what they tried, a quota's users", + Long: `Top lists over the last while: + + devices inside addresses by traffic to and from external zones (needs + hc settings set --stats-per-device on) + blocked sources of logged drops (the policies' drops with 'log drops' + on, and rules that log) + tried what those drops tried: tcp/22, udp/53, icmp/echo-request + quota ID who used a per-address quota of rule ID`, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + get1 := func(kind string) ([]statEntry, error) { + return get[[]statEntry]("/stats/top", q("kind", kind, "since", since, "limit", strconv.Itoa(limit))) + } + switch args[0] { + case "devices": + up, err := get1("device_up") + if err != nil { + return err + } + down, err := get1("device_down") + if err != nil { + return err + } + out := map[string][2]int64{} + var order []string + for _, l := range [][]statEntry{down, up} { + for _, e := range l { + if _, ok := out[e.Key]; !ok { + order = append(order, e.Key) + } + } + } + for _, e := range down { + v := out[e.Key] + v[0] = e.Count + out[e.Key] = v + } + for _, e := range up { + v := out[e.Key] + v[1] = e.Count + out[e.Key] = v + } + return output(map[string]any{"up": up, "down": down}, func(w io.Writer) { + fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT") + for _, k := range order { + fmt.Fprintf(w, "%s\t%s\t%s\n", k, humanBytes(out[k][0]), humanBytes(out[k][1])) + } + }) + case "blocked", "tried": + kind := map[string]string{"blocked": "blocked_src", "tried": "blocked_port"}[args[0]] + es, err := get1(kind) + if err != nil { + return err + } + return output(es, func(w io.Writer) { + fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS") + for _, e := range es { + fmt.Fprintf(w, "%s\t%d\n", e.Key, e.Count) + } + }) + case "quota": + if len(args) != 2 { + return fmt.Errorf("give the rule's ID: hc stats top quota 12") + } + es, err := get1("quota_r" + args[1]) + if err != nil { + return err + } + return output(es, func(w io.Writer) { + fmt.Fprintln(w, "ADDRESS\tUSED") + for _, e := range es { + fmt.Fprintf(w, "%s\t%s\n", e.Key, humanBytes(e.Count)) + } + }) + } + return fmt.Errorf("unknown list %q: devices, blocked, tried or quota", args[0]) + }, + } + cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…") + cmd.Flags().IntVar(&limit, "limit", 20, "at most this many") + return cmd +} + +func statsRulesCmd() *cobra.Command { + var since string + var unused bool + cmd := &cobra.Command{ + Use: "rules", Short: "Packets each rule, built-in and policy counted; --unused: rules that matched nothing", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + es, err := get[[]statEntry]("/stats/hits", q("since", since)) + if err != nil { + return err + } + if unused { + var keep []statEntry + for _, e := range es { + if e.Count == 0 && strings.HasPrefix(e.Key, "r") { + keep = append(keep, e) + } + } + es = keep + } + return output(es, func(w io.Writer) { + if unused && len(es) == 0 { + fmt.Fprintf(w, "every rule matched something in the last %s\n", since) + return + } + fmt.Fprintln(w, "WHAT\tPACKETS") + for _, e := range es { + fmt.Fprintf(w, "%s\t%d\n", orDash(e.Name), e.Count) + } + }) + }, + } + cmd.Flags().StringVar(&since, "since", "30d", "the last 24h, 7d, 30d…") + cmd.Flags().BoolVar(&unused, "unused", false, "only the rules that matched nothing") + return cmd +} + +func statsConnectionsCmd() *cobra.Command { + var since, step string + cmd := &cobra.Command{ + Use: "connections", Short: "How many connections the firewall tracked over time, against the limit", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", "g:conntrack,g:conntrack_max")) + if err != nil { + return err + } + return output(tl, func(w io.Writer) { + fmt.Fprintln(w, "TIME\tCONNECTIONS\tLIMIT") + for i, t := range tl.Times { + n, m := tl.Series[0].Values[i], tl.Series[1].Values[i] + if n < 0 { + continue // no sample + } + fmt.Fprintf(w, "%s\t%d\t%d\n", time.Unix(t, 0).Local().Format("01-02 15:04"), n, m) + } + }) + }, + } + cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…") + cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d") + return cmd +} diff --git a/cmd/housecarld/main.go b/cmd/housecarld/main.go index 9163feb..2fa9a16 100644 --- a/cmd/housecarld/main.go +++ b/cmd/housecarld/main.go @@ -35,6 +35,7 @@ import ( "housecarl/internal/nft" "housecarl/internal/render" "housecarl/internal/server" + "housecarl/internal/stats" "housecarl/internal/store" "housecarl/internal/trace" "housecarl/internal/validate" @@ -198,6 +199,7 @@ func serveCmd() *cobra.Command { go nflog.Listen(ctx, render.LogGroup, a.log, srv.LogPacket) go srv.RunQuotas(ctx) srv.SetConntrack(conntrack.Client{}) + statsDone := startStats(ctx, a, fw, srv) // A trace left behind by a crash. fw.DeleteTable(ctx, "inet", trace.Table) hup := make(chan os.Signal, 1) @@ -212,12 +214,45 @@ func serveCmd() *cobra.Command { err = srv.Serve(ctx, func() { sdNotify("READY=1") }) sdNotify("STOPPING=1") srv.SaveQuotaUsage(context.Background()) + <-statsDone a.log.Info("stopped") return err }, } } +// startStats opens the statistics database and runs the collector until +// ctx ends; the channel closes once it wrote its last figures. +func startStats(ctx context.Context, a *app, fw *nft.Firewall, srv *server.Server) <-chan struct{} { + done := make(chan struct{}) + c := a.cfg.Stats + if !c.Enabled { + close(done) + return done + } + st, err := stats.OpenStore(c.Database) + if err != nil { + a.log.Error("statistics are off: opening their database failed (deleting it is safe)", "err", err) + close(done) + return done + } + opts := stats.Options{TopN: c.TopN, RetainMinute: time.Duration(c.Retention.Minute), RetainHour: time.Duration(c.Retention.Hour), RetainDay: time.Duration(c.Retention.Day)} + // Tests sample more often than every minute. + if v := os.Getenv("HOUSECARL_STATS_INTERVAL"); v != "" { + opts.Interval, _ = time.ParseDuration(v) + } + src := stats.Sources{Table: fw.Table} + srv.StatsSources(&src) + col := stats.NewCollector(st, opts, src, a.log) + srv.SetStats(col) + go func() { + defer close(done) + defer st.Close() + col.Run(ctx) + }() + return done +} + func sdNotify(state string) { sock := os.Getenv("NOTIFY_SOCKET") if sock == "" { diff --git a/docs/api.md b/docs/api.md index 2f088b4..cc6438c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -99,7 +99,8 @@ Creating answers 201, other changes 200. "builtins": {"loopback": true, "established": true, "drop_invalid": true, "icmpv6": true, "ping_internal": true, "dhcpv6_client": true, "antispoof": true, "mss_clamp": false}, "own_prefixes": ["2001:db8::/48"], "antispoof_v4": ["10.0.0.0/8", "..."], "antispoof_v6": ["fc00::/7", "..."], - "flowtable_devices": ["wan", "eth1"], "probation_seconds": 120, "manual_confirm": false} + "flowtable_devices": ["wan", "eth1"], "probation_seconds": 120, "manual_confirm": false, + "stats_per_device": false} ``` Lists replace the old ones. `probation_seconds` 0 is the default (120). @@ -145,6 +146,12 @@ A source NAT rule is a match (as a rule's, without the rule's other fields) with `"to"` (empty: masquerade), a redirect a match with `"to_port"`. Times are seconds (`timeout`). +### Statistics + +`GET /stats/status`, `/stats/summary`, `/stats/timeline`, `/stats/top`, +`/stats/hits` and `/stats/heatmap` (`log:read`): see +[stats.md](stats.md#the-api). + ### Changesets Changes staged to be reviewed and applied together ([design.md](design.md#reviewing-changes-changesets)). diff --git a/docs/cli.md b/docs/cli.md index 63a76c1..aac9906 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -138,6 +138,7 @@ by name. | `quotas` | what each rule's quota used (each address's, for per-address quotas), and when it resets | | `log [-f] [--search TEXT] [--limit N]` | the packets logged by rules and policies ([operations.md](operations.md#the-log)) | | `connections [--search TEXT] [--limit N]`, `connections close --search TEXT [-y]` | tracked connections; closing makes the rules check them again ([operations.md](operations.md#connections)) | +| `stats [--since 24h]` | traffic per zone, drops, the busiest rules, blocked sources, devices; `stats status`, `stats traffic`, `stats top devices\|blocked\|tried\|quota ID`, `stats rules [--unused]`, `stats connections` ([stats.md](stats.md#the-cli)) | | `matrix [FROM TO]` | what the rules do to new connections between zones; with FROM and TO, that pair's rules in order ([rules.md](rules.md#the-zone-matrix)) | | `trace [--address A] [--port N] [--protocol P] [--seconds S]` | follow packets through every table's rules ([operations.md](operations.md#tracing)) | @@ -207,7 +208,7 @@ See [nat.md](nat.md). | Command | | |---|---| | `settings show` | | -| `settings set [--input-policy\|--forward-policy\|--output-policy accept\|drop] [--log-drops on\|off] [--builtin NAME=on\|off]... [--own-prefix P]... [--antispoof-v4 P]... [--antispoof-v6 P]... [--flowtable DEV]... [--probation 2m] [--auto-confirm on\|off]` | lists given replace the old ones | +| `settings set [--input-policy\|--forward-policy\|--output-policy accept\|drop] [--log-drops on\|off] [--builtin NAME=on\|off]... [--own-prefix P]... [--antispoof-v4 P]... [--antispoof-v6 P]... [--flowtable DEV]... [--probation 2m] [--auto-confirm on\|off] [--stats-per-device on\|off]` | lists given replace the old ones | ### probation, history, import diff --git a/docs/configuration.md b/docs/configuration.md index a4fa076..3b7ca93 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,11 +26,16 @@ reads the file when it starts. | `backup.dir` | `/var/backups/housecarl` | where `housecarld backup` writes | | `backup.retention` | `30d` | backups older than this are deleted… | | `backup.keep_at_least` | `7` | …keeping at least this many | +| `stats.enabled` | `true` | keep traffic, drops, hits, blocked sources, connections and quotas over time ([stats.md](stats.md)) | +| `stats.database` | `/var/lib/housecarl/stats.db` | the statistics' own database; deleting it is safe | +| `stats.retention.minute`, `.hour`, `.day` | `7d`, `90d`, `730d` | how long per-minute, hourly and daily figures are kept | +| `stats.top_n` | `100` | entries kept per top list, per hour and per day | Durations are written `15s`, `12h` or `30d`. The probation's length is a firewall setting, not here: `hc settings set ---probation 5m`. +--probation 5m`. So is counting traffic per device for the statistics: +`hc settings set --stats-per-device on`. ## HTTPS diff --git a/docs/stats.md b/docs/stats.md new file mode 100644 index 0000000..91df661 --- /dev/null +++ b/docs/stats.md @@ -0,0 +1,140 @@ +# Statistics + +Housecarl keeps what the firewall did over time: + +- traffic per interface and zone, received and sent; +- packets dropped, by chain, and when drops happen (a weekday × hour + heatmap); +- hits per rule, built-in and policy, and the rules that matched nothing; +- the sources blocked most and what they tried (from the log); +- traffic per inside device (a setting, below); +- connections tracked, against the kernel's limit; +- what each quota counted; +- housecarld's and the machine's memory, CPU and load. + +All of it is in the web UI (**Traffic → Statistics**, and two charts on +the dashboard), the CLI (`hc stats`) and the API (`/api/v1/stats/...`), +for anyone with `log:read`. + +Statistics are **on by default**. They cost nothing on the packets' +path: housecarld reads counters the kernel keeps anyway, once a minute. +Only traffic per device adds to the ruleset, so it is a setting you turn +on. The figures are kept as aggregates in a database of their own, +`/var/lib/housecarl/stats.db`, separate from the configuration. Deleting +it is always safe (stop housecarl first), and backups leave it out. + +## Settings + +In `/etc/housecarl/housecarl.yaml` (restart housecarl after a change): + +```yaml +stats: + enabled: true + database: /var/lib/housecarl/stats.db + retention: + minute: 7d # per-minute figures + hour: 90d # hourly figures and top lists + day: 730d # daily figures and top lists + top_n: 100 # entries kept per top list, per hour and day +``` + +Old data is removed once an hour. A range picks the finest data that +covers it: per minute up to 2 days, hourly up to 90 days, then daily. + +## Traffic per device + +```sh +hc settings set --stats-per-device on +``` + +(or **Settings → Statistics** in the web UI). This is a firewall setting, +so it goes through the usual review and probation. Housecarl then adds +four sets to its table, and four rules at the top of the forward chain +(shown as the built-in `stats`): + +``` +iifname != "wan" oifname "wan" update @stats_up_v4 { ip saddr } +iifname != "wan" oifname "wan" update @stats_up_v6 { ip6 saddr and ffff:ffff:ffff:ffff:: } +iifname "wan" oifname != "wan" update @stats_down_v4 { ip daddr } +iifname "wan" oifname != "wan" update @stats_down_v6 { ip6 daddr and ffff:ffff:ffff:ffff:: } +``` + +Each inside address talking to an external zone gets a counter: what it +sent out, and what came back to it. IPv6 addresses are counted per /64, +since one device uses many addresses. Counting is done in the kernel, a +set lookup per forwarded packet. An address quiet for an hour drops out +of the sets, and a reload keeps the counts. + +With the flowtable on, established connections skip the forward chain, +so only their first packets are counted (warning `stats_flowtable`). Set +names starting with `stats_` (and `meter_`, `conn_`, `quota_`) are +Housecarl's own and can't be used for your sets. + +## What is collected + +Every minute housecarld samples its sources and stores what changed: + +| Figure | Source | +|---|---| +| `rx:IF`, `tx:IF` (bytes), `rxp:IF`, `txp:IF` (packets) | the interfaces' counters (`/sys/class/net`); the loopback is left out | +| `zrx:ZONE`, `ztx:ZONE` | the same, summed per zone: received from and sent to the zone's interfaces | +| `hits:MARK`, `hitb:MARK` | each rule's (`r12`), built-in's (`b-established`), port forward's (`f3`) and policy's (`p-input`) counter, in packets and bytes | +| `drop:CHAIN`, `drops` | the packets counted by what drops them: rules that drop or reject, the drop policies, antispoofing (`early`) and invalid packets (`invalid`) | +| `quota:r12` | bytes counted against a rule's quota | +| `g:conntrack`, `g:conntrack_max` | connections tracked, and the limit (`nf_conntrack_max`) | +| `g:rss`, `g:cpu` | housecarld's memory (bytes) and CPU (tenths of a percent of one CPU) | +| `g:mem`, `g:load` | the host's memory in use, and its 1-minute load average × 100 | + +`g:` figures are gauges: the API averages them over a bucket. + +Top lists, per hour and per day, the `top_n` largest: + +| List | | +|---|---| +| `blocked_src` | sources of logged drops | +| `blocked_port` | what they tried: `tcp/22`, `udp/53`, `icmp/echo-request` | +| `device_up`, `device_down` | bytes per inside address, sent and received (traffic per device) | +| `quota_r12` | a per-address quota's use, per address | + +The blocked lists only see what is **logged**: the input and forward +policies' drops with `hc settings set --log-drops on`, and rules with +logging. The drop counts above see everything. + +A counter that goes down (an interface that came back, a quota reset) +counts again from zero. What happens while housecarld is stopped isn't +counted. + +## The CLI + +```sh +hc stats --since 7d # summary: traffic per zone, drops, busiest rules, top lists +hc stats status # collecting? since when, the database, figures now +hc stats traffic --since 24h # per zone over time (--interfaces: per interface) +hc stats top devices --since 30d # devices by traffic (blocked, tried, quota RULE-ID) +hc stats rules --unused # rules that matched nothing in 30 days +hc stats connections --since 24h +``` + +Ranges are written `90m`, `24h`, `7d` or `4w`. `-o json` gives the API's +answer. + +## The API + +| Endpoint | | +|---|---| +| `GET /stats/status` | `enabled`, `database`, `size`, `first_data`, `last_sample`, `sample_error`, `per_device`, `retention`, `latest` (the newest gauges) | +| `GET /stats/summary?since=24h&limit=10` | `zones` and `interfaces` (`name`, `received`, `sent`), `drops`, `drops_by_chain`, `rules` (busiest, named), `peak_connections`, `conntrack_max`, `blocked_sources`, `blocked_ports`, `devices_up`, `devices_down` | +| `GET /stats/timeline?keys=zrx:*,drops&since=24h&step=auto` | `step`, `times` (bucket starts), `series` (`key`, `values`); `GROUP:*` means every key of a group; `step` is `auto`, `1m`, `5m`, `15m`, `1h`, `6h` or `1d`; a gauge without a sample is -1 | +| `GET /stats/top?kind=device_down&since=7d&limit=20` | a top list (`key`, `count`) | +| `GET /stats/hits?since=30d` | packets per rule, built-in and policy (`key`, `count`, `name`), with the enabled rules that counted none | +| `GET /stats/heatmap?key=drops&since=28d` | `avg[weekday][hour]` (Monday first, local time; -1 where there is no data) | + +With statistics off, `/stats/status` says `enabled: false` and the others +answer 503. + +## Warnings + +When the connection tracking table is over 80% full, the status shows +the warning `conntrack_full`: once it is full, the kernel drops new +connections. Raise `net.netfilter.nf_conntrack_max`, or find what opens +so many on the Connections page. diff --git a/docs/validation.md b/docs/validation.md index 7eb3ae5..eea4046 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -34,6 +34,8 @@ are unique per kind, ignoring case. | `flowtable_device` | error | a flowtable device with a wildcard: name each device | | `flowtable_device_missing` | warning, *host* | a flowtable device doesn't exist; the flowtable is left out until it does | | `probation_range` | error | the probation isn't 10 to 3600 seconds | +| `stats_no_external` | warning | traffic per device is on, but no zone is external: it counts what goes to and comes from external zones | +| `stats_flowtable` | warning | traffic per device with the flowtable on: its fast path skips the forward chain, so only each connection's first packets are counted | ## Zones @@ -49,6 +51,7 @@ are unique per kind, ignoring case. | Code | | When | |---|---|---| +| `set_name_reserved` | error | a set name starting with `meter_`, `conn_`, `quota_` or `stats_`: Housecarl's own sets (rate limits, quotas, statistics) use those | | `set_entry` | error | an entry that isn't an address, prefix (without host bits) or range | | `set_empty` | warning | a set (not dynamic) with no entries: rules using it match nothing | | `set_dynamic_entries` | error | configured entries on a dynamic set: its entries come at run time | @@ -127,4 +130,5 @@ configuration: the dashboard and `hc status` show them. | Code | | When | |---|---|---| | `nftables_service` | warning, *host* | Debian's `nftables.service` is enabled: at boot it loads `/etc/nftables.conf`, and stopping or restarting it runs `nft flush ruleset`, deleting Housecarl's table too. Disable it with `systemctl disable nftables` (without `--now`) | +| `conntrack_full` | warning, *host* | the connection tracking table is over 80% full; once it is full, the kernel drops new connections. Raise `net.netfilter.nf_conntrack_max`, or find what opens so many ([stats.md](stats.md#warnings)) | | `takeover_pending` | warning, *host* | the table you imported from is still in force beside Housecarl's, so a packet must get past both; `hc takeover TABLE` deletes it once Housecarl's rules are right | diff --git a/internal/config/config.go b/internal/config/config.go index cf35026..5d18dd2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,25 @@ type Config struct { // Ruleset is where the confirmed ruleset is saved for the boot unit // (housecarl-ruleset.service loads it before the network comes up). Ruleset string `yaml:"ruleset"` + // Stats keeps traffic, hits and drops over time (docs/stats.md). + Stats Stats `yaml:"stats"` +} + +// Stats configures statistics: a database of their own, aggregates +// only, pruned by age. +type Stats struct { + Enabled bool `yaml:"enabled"` + Database string `yaml:"database"` + Retention StatsRetention `yaml:"retention"` + // TopN is how many entries the top lists keep per hour and day. + TopN int `yaml:"top_n"` +} + +// StatsRetention is how long each resolution is kept. +type StatsRetention struct { + Minute Duration `yaml:"minute"` + Hour Duration `yaml:"hour"` + Day Duration `yaml:"day"` } // TLS is the web UI's certificate: from files, or obtained through ACME @@ -183,7 +202,9 @@ func Default() *Config { Log: Log{Level: "info", Format: "text"}, Session: Session{Lifetime: Duration(12 * time.Hour), IdleTimeout: Duration(2 * time.Hour), CookieSecure: "auto"}, Backup: Backup{Dir: "/var/backups/housecarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7}, - TLS: TLS{ACME: ACME{Directory: "letsencrypt-staging", Resolvers: []string{"1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"}, CheckPropagation: true}}, + Stats: Stats{Enabled: true, Database: "/var/lib/housecarl/stats.db", TopN: 100, + Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)}}, + TLS: TLS{ACME: ACME{Directory: "letsencrypt-staging", Resolvers: []string{"1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"}, CheckPropagation: true}}, } } @@ -393,4 +414,17 @@ backup: dir: /var/backups/housecarl retention: 30d keep_at_least: 7 + +# Statistics (docs/stats.md): traffic per interface and zone, hits and +# drops, blocked sources, connections, quotas, kept over time in a database +# of their own (aggregates only). Deleting it is always safe. Traffic per +# device is a firewall setting: hc settings set --stats-per-device on. +stats: + enabled: true + database: /var/lib/housecarl/stats.db + retention: + minute: 7d # per-minute figures + hour: 90d # hourly figures and top lists + day: 730d # daily figures and top lists + top_n: 100 # entries kept per top list, per hour and day ` diff --git a/internal/model/settings.go b/internal/model/settings.go index 6eca9d8..73c9c25 100644 --- a/internal/model/settings.go +++ b/internal/model/settings.go @@ -36,6 +36,10 @@ type Settings struct { // itself once it reaches the API again over a new connection, from the // same address: proof that it can still manage the firewall. ManualConfirm bool `json:"manual_confirm,omitempty"` + + // StatsPerDevice counts forwarded traffic to and from external zones + // per inside address (IPv6 per /64), for the statistics' top devices. + StatsPerDevice bool `json:"stats_per_device,omitempty"` } // Builtins are rules Housecarl adds itself, shown as "added by Housecarl". diff --git a/internal/nft/nft.go b/internal/nft/nft.go index 73fa82f..3ba848e 100644 --- a/internal/nft/nft.go +++ b/internal/nft/nft.go @@ -201,6 +201,9 @@ type Table struct { // Quotas: the rules' quotas' used bytes (quota_r12), and the // per-address quotas' entries by set (quota_r12_v4). QuotaUsage + // DeviceSets are the per-device traffic counters by set + // (stats_up_v4, stats_down_v6 …): each address's packets and bytes. + DeviceSets map[string][]render.Element // Since is when the counters started (housecarld's first apply). Since time.Time // Tables are all the machine's tables ("inet main"), when the whole @@ -287,7 +290,7 @@ func (q jsonQuota) bytes() int64 { // ParseTable reads `nft -j list table inet housecarl`. func ParseTable(data []byte) (*Table, error) { - t := &Table{Counters: map[string]Counter{}, Elements: map[string][]render.Element{}, + t := &Table{Counters: map[string]Counter{}, Elements: map[string][]render.Element{}, DeviceSets: map[string][]render.Element{}, QuotaUsage: QuotaUsage{Quotas: map[string]int64{}, QuotaSets: map[string][]render.Element{}}} if len(data) == 0 { return t, nil @@ -340,6 +343,21 @@ func ParseTable(data []byte) (*Table, error) { } continue } + if render.IsDeviceSet(s.Name) { + for _, e := range s.Elem { + var w struct { + Elem struct { + Val string `json:"val"` + Expires int `json:"expires"` + Counter *Counter `json:"counter"` + } `json:"elem"` + } + if json.Unmarshal(e, &w) == nil && w.Elem.Counter != nil { + t.DeviceSets[s.Name] = append(t.DeviceSets[s.Name], render.Element{Addr: w.Elem.Val, Timeout: w.Elem.Expires, Packets: w.Elem.Counter.Packets, Bytes: w.Elem.Counter.Bytes}) + } + } + continue + } name := strings.TrimSuffix(strings.TrimSuffix(s.Name, "_v4"), "_v6") if name == s.Name || strings.HasPrefix(s.Name, "meter_") || strings.HasPrefix(s.Name, "conn_") { continue @@ -457,6 +475,9 @@ func (f *Firewall) Apply(ctx context.Context, st *model.State) error { } usage.merge(f.restore) opt.Quotas, opt.QuotaSets = usage.Quotas, usage.QuotaSets + if old != nil { + opt.DeviceSets = old.DeviceSets + } if err := f.Runner.Apply(ctx, render.Render(st, opt)); err != nil { return err } diff --git a/internal/nft/nft_test.go b/internal/nft/nft_test.go index dc0eff8..d7f8d43 100644 --- a/internal/nft/nft_test.go +++ b/internal/nft/nft_test.go @@ -208,3 +208,52 @@ func TestQuotasInContainer(t *testing.T) { t.Errorf("after a reboot: total %d, per destination %d (saved %+v)", t4, d4, saved) } } + +// TestDeviceCountsInContainer: the per-device counters are read back and +// survive a reload. +func TestDeviceCountsInContainer(t *testing.T) { + image := os.Getenv("HOUSECARL_NFT_IMAGE") + if image == "" { + t.Skip("HOUSECARL_NFT_IMAGE not set") + } + name := "housecarl-nft-stats-test" + exec.Command("docker", "rm", "-f", name).Run() + if out, err := exec.Command("docker", "run", "-d", "--name", name, "--cap-add", "NET_ADMIN", image, "sleep", "300").CombinedOutput(); err != nil { + t.Fatalf("starting the container: %v %s", err, out) + } + t.Cleanup(func() { exec.Command("docker", "rm", "-f", name).Run() }) + fw := &Firewall{Runner: &Runner{Command: []string{"docker", "exec", "-i", name, "nft"}}} + ctx := context.Background() + st := model.NewState() + st.Zones = []*model.Zone{{ID: 1, Name: "wan", Interfaces: []string{"eth0"}, External: true}} + st.Settings.StatsPerDevice = true + if err := fw.Apply(ctx, st); err != nil { + t.Fatalf("apply: %v", err) + } + if out, err := exec.Command("docker", "exec", name, "nft", "add", "element", "inet", "housecarl", "stats_up_v4", "{ 10.0.10.5 counter packets 7 bytes 7000 }").CombinedOutput(); err != nil { + t.Fatalf("adding an element: %v %s", err, out) + } + read := func() render.Element { + tb, err := fw.Table(ctx) + if err != nil { + t.Fatal(err) + } + for _, e := range tb.DeviceSets["stats_up_v4"] { + if e.Addr == "10.0.10.5" { + return e + } + } + t.Fatalf("no 10.0.10.5 in %+v", tb.DeviceSets) + return render.Element{} + } + if e := read(); e.Packets != 7 || e.Bytes != 7000 || e.Timeout <= 0 { + t.Fatalf("read back %+v", e) + } + st.Settings.InputPolicy = model.PolicyAccept // a change: a reload + if err := fw.Apply(ctx, st); err != nil { + t.Fatal(err) + } + if e := read(); e.Packets != 7 || e.Bytes != 7000 { + t.Errorf("after a reload: %+v", e) + } +} diff --git a/internal/render/render.go b/internal/render/render.go index 3ce231e..8907383 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -39,6 +39,9 @@ type Options struct { // quota_r12_v4), so a reload keeps what was used. Quotas map[string]int64 QuotaSets map[string][]Element + // DeviceSets are the per-device traffic counters' entries + // (stats_up_v4 …), so a reload keeps them. + DeviceSets map[string][]Element } // Element is a dynamic set's entry with its remaining lifetime, or a @@ -47,6 +50,9 @@ type Element struct { Addr string `json:"address"` Timeout int `json:"timeout,omitempty"` // seconds left (0: none) Used int64 `json:"used,omitempty"` // bytes + // Packets and Bytes are a per-device counter's. + Packets uint64 `json:"packets,omitempty"` + Bytes uint64 `json:"bytes,omitempty"` } // QuotaName is the name of a rule's quota: the named quota of a total @@ -110,6 +116,7 @@ func (r *renderer) render() { w.emit("", "delete table inet %s", Table) w.emit("", "table inet %s {", Table) r.sets() + r.deviceSets() devices := r.flowtableDevices() if len(devices) > 0 { w.emit("built-in: flowtable", "\tflowtable ft { hook ingress priority filter; devices = { %s }; }", strings.Join(quoteAll(devices), ", ")) @@ -190,6 +197,56 @@ func (r *renderer) elements() { } } +// IsDeviceSet reports whether a set is a per-device traffic counter. +func IsDeviceSet(name string) bool { + return strings.HasPrefix(name, "stats_up_") || strings.HasPrefix(name, "stats_down_") +} + +// deviceSets declares the per-device traffic counters, with what they +// counted before a reload. +func (r *renderer) deviceSets() { + if !r.st.Settings.StatsPerDevice || len(r.st.ExternalInterfaces()) == 0 { + return + } + for _, dir := range []string{"up", "down"} { + for _, v6 := range []bool{false, true} { + name := setName("stats_"+dir, v6) + var elems []string + for _, e := range r.opt.DeviceSets[name] { + a, err := netip.ParseAddr(e.Addr) + if err != nil || a.Is6() != v6 { + continue + } + timeout := "" + if e.Timeout > 0 { + timeout = fmt.Sprintf(" timeout %ds", e.Timeout) + } + elems = append(elems, fmt.Sprintf("%s%s counter packets %d bytes %d", a, timeout, e.Packets, e.Bytes)) + } + list := "" + if len(elems) > 0 { + list = " elements = { " + strings.Join(elems, ", ") + " }" + } + r.w.emit("built-in: stats", "\tset %s { type %s; flags dynamic,timeout; timeout 1h; size 65535; counter;%s }", name, addrType(v6), list) + } + } +} + +// deviceCounts counts forwarded traffic per inside address: what leaves +// for an external zone by its source, what comes from one by its +// destination (IPv6 by /64). +func (r *renderer) deviceCounts() { + ext := r.st.ExternalInterfaces() + if !r.st.Settings.StatsPerDevice || len(ext) == 0 { + return + } + e := ifaces(ext) + r.builtin("stats", "iifname != %s oifname %s update @stats_up_v4 { ip saddr }", e, e) + r.builtin("stats", "iifname != %s oifname %s update @stats_up_v6 { ip6 saddr and ffff:ffff:ffff:ffff:: }", e, e) + r.builtin("stats", "iifname %s oifname != %s update @stats_down_v4 { ip daddr }", e, e) + r.builtin("stats", "iifname %s oifname != %s update @stats_down_v6 { ip6 daddr and ffff:ffff:ffff:ffff:: }", e, e) +} + // SetName is the nftables name of a set's family part. func SetName(name string, v6 bool) string { return setName(name, v6) } @@ -313,6 +370,7 @@ func (r *renderer) forward(flowtable bool) { r.builtin("mss-clamp", "oifname %s tcp flags syn tcp option maxseg size set rt mtu", ifaces(ext)) } r.rules(model.ChainForward, true) + r.deviceCounts() if flowtable { r.builtin("flowtable", "meta l4proto { tcp, udp } flow add @ft") } diff --git a/internal/render/render_test.go b/internal/render/render_test.go index bfb443c..c88d4d1 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -206,6 +206,31 @@ func TestRenderRouter(t *testing.T) { } } +func TestRenderDeviceCounts(t *testing.T) { + st := router() + st.Settings.StatsPerDevice = true + text := Render(st, Options{DeviceSets: map[string][]Element{"stats_up_v4": {{Addr: "10.0.10.5", Timeout: 3000, Packets: 7, Bytes: 7000}}}}).Text + for _, want := range []string{ + `set stats_up_v4 { type ipv4_addr; flags dynamic,timeout; timeout 1h; size 65535; counter; elements = { 10.0.10.5 timeout 3000s counter packets 7 bytes 7000 } }`, + `set stats_down_v6 { type ipv6_addr; flags dynamic,timeout; timeout 1h; size 65535; counter; }`, + `iifname != "wan" oifname "wan" update @stats_up_v4 { ip saddr } comment "hc:b-stats"`, + `iifname "wan" oifname != "wan" update @stats_down_v6 { ip6 daddr and ffff:ffff:ffff:ffff:: } comment "hc:b-stats"`, + } { + if !strings.Contains(text, want) { + t.Errorf("missing %q", want) + } + } + // Counted before the established built-in, so every packet is. + fwd := text[strings.Index(text, "chain forward {"):] + if i := strings.Index(fwd, "update @stats_up_v4"); i < 0 || i > strings.Index(fwd, "hc:b-established") { + t.Error("the counting comes after the established built-in") + } + st.Settings.StatsPerDevice = false + if strings.Contains(Render(st, Options{}).Text, "stats_") { + t.Error("counting without the setting") + } +} + // TestRenderedRulesetPassesNft has the real nft check the router's // ruleset (and a minimal one). Set HOUSECARL_NFT to how nft is run, e.g. // @@ -217,9 +242,12 @@ func TestRenderedRulesetPassesNft(t *testing.T) { if len(cmd) == 0 { t.Skip("HOUSECARL_NFT not set") } - for name, st := range map[string]*model.State{"router": router(), "empty": model.NewState()} { + stats := router() + stats.Settings.StatsPerDevice = true + for name, st := range map[string]*model.State{"router": router(), "empty": model.NewState(), "stats": stats} { q, qs := quotaOptions(st) - rs := Render(st, Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}, {Addr: "2a02:8013:7f3d:30::5"}}}, Quotas: q, QuotaSets: qs}) + rs := Render(st, Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}, {Addr: "2a02:8013:7f3d:30::5"}}}, Quotas: q, QuotaSets: qs, + DeviceSets: map[string][]Element{"stats_up_v4": {{Addr: "10.0.10.5", Timeout: 3000, Packets: 7, Bytes: 7000}}, "stats_down_v6": {{Addr: "2001:db8:1:2::", Packets: 1, Bytes: 100}}}}) c := exec.CommandContext(context.Background(), cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...) c.Stdin = strings.NewReader(rs.Text) if out, err := c.CombinedOutput(); err != nil { diff --git a/internal/server/h_stats.go b/internal/server/h_stats.go new file mode 100644 index 0000000..408348a --- /dev/null +++ b/internal/server/h_stats.go @@ -0,0 +1,266 @@ +package server + +import ( + "net/http" + "strconv" + "strings" + + "housecarl/internal/api" + "housecarl/internal/auth" + "housecarl/internal/model" + "housecarl/internal/stats" + "housecarl/internal/validate" +) + +// Statistics: the firewall's figures over time (internal/stats). + +// StatsSources fills in what the statistics collector needs from the +// server: the interfaces' zones, and which counters count drops. +func (s *Server) StatsSources(src *stats.Sources) { + src.Zones = s.interfaceZones + src.Classify = s.classifyMark +} + +// SetStats gives the server the statistics collector (nil: statistics +// are off). +func (s *Server) SetStats(c *stats.Collector) { s.stats = c } + +// interfaceZones maps the machine's interfaces to their zones. +func (s *Server) interfaceZones() map[string]string { + out := map[string]string{} + for _, i := range s.hostInterfaces(s.eng.State()) { + if i.Zone != "" { + out[i.Name] = i.Zone + } + } + return out +} + +// classifyMark says which chain a counter is in and whether what it +// counts is dropped: rules that drop or reject, the drop policies, and +// the built-ins that drop (antispoofing, invalid packets). +func (s *Server) classifyMark(mark string) (string, bool) { + st := s.eng.State() + switch { + case strings.HasPrefix(mark, "p-"): + chain := strings.TrimPrefix(mark, "p-") + policy := map[string]string{"input": st.Settings.InputPolicy, "forward": st.Settings.ForwardPolicy, "output": st.Settings.OutputPolicy}[chain] + return chain, policy == model.PolicyDrop + case strings.HasPrefix(mark, "b-antispoof"): + return model.ChainEarly, true + case mark == "b-invalid": + return "invalid", true + case strings.HasPrefix(mark, "r"): + num := strings.TrimLeft(mark[1:], "0123456789") + id, _ := strconv.ParseInt(strings.TrimSuffix(mark[1:], num), 10, 64) + if o, ok := st.ByID(model.KindRule, id).(*model.Rule); ok && o != nil { + return o.Chain, o.Action == model.ActionDrop || o.Action == model.ActionReject + } + } + return "", false +} + +// markName names a counter's object: "rule input #12 (ssh)", "built-in: +// established", "input policy". +func markName(st *model.State, mark string) string { + if c, ok := strings.CutPrefix(mark, "p-"); ok { + return c + " policy" + } + if l := markLabel(st, api.TraceStep{Kind: "rule", Mark: mark}); l != "" { + return l + } + if len(mark) < 2 { + return mark + } + // An object deleted since it counted. + kind := map[byte]string{'r': "rule", 'f': "port forward", 's': "source NAT", 'd': "redirect"}[mark[0]] + if num := strings.TrimLeft(mark[1:], "0123456789"); kind != "" && len(num) < len(mark)-1 { + return kind + " #" + strings.TrimSuffix(mark[1:], num) + " (deleted)" + } + return mark +} + +func (s *Server) statsCollector() (*stats.Collector, error) { + if s.stats == nil { + return nil, &httpError{http.StatusServiceUnavailable, "stats_off", "statistics are off: set stats.enabled in housecarl.yaml and restart housecarld"} + } + return s.stats, nil +} + +func (s *Server) statsQuerier(p *auth.Principal) (*stats.Querier, error) { + if err := require(p, auth.ResLog, false); err != nil { + return nil, err + } + c, err := s.statsCollector() + if err != nil { + return nil, err + } + q := stats.NewQuerier(c) + q.Now = s.nowFn + return q, nil +} + +func since(r *http.Request, def string) (string, error) { + v := r.URL.Query().Get("since") + if v == "" { + v = def + } + if _, err := stats.ParseSince(v); err != nil { + return "", badRequest("%v", err) + } + return v, nil +} + +// StatsStatus is the collector's health, with the setting that counts +// traffic per device. +type StatsStatus struct { + stats.Status + Latest *stats.Latest `json:"latest,omitempty"` +} + +func (s *Server) statsStatus(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := require(p, auth.ResLog, false); err != nil { + return err + } + out := StatsStatus{} + if s.stats != nil { + out.Status = s.stats.Status() + out.Latest = s.stats.Latest() + if !p.IsAdmin() { + out.Database = "" + } + } + out.PerDevice = s.eng.State().Settings.StatsPerDevice + writeJSON(w, http.StatusOK, out) + return nil +} + +func (s *Server) statsSummary(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + q, err := s.statsQuerier(p) + if err != nil { + return err + } + v, err := since(r, "24h") + if err != nil { + return err + } + d, _ := stats.ParseSince(v) + sum, err := q.Summary(r.Context(), d, queryInt(r, "limit", 10)) + if err != nil { + return err + } + st := s.eng.State() + for i := range sum.Rules { + sum.Rules[i].Name = markName(st, sum.Rules[i].Key) + } + writeJSON(w, http.StatusOK, sum) + return nil +} + +func (s *Server) statsTimeline(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + q, err := s.statsQuerier(p) + if err != nil { + return err + } + v, err := since(r, "24h") + if err != nil { + return err + } + d, _ := stats.ParseSince(v) + var keys []string + for _, k := range strings.Split(r.URL.Query().Get("keys"), ",") { + if k = strings.TrimSpace(k); k != "" { + keys = append(keys, k) + } + } + if len(keys) == 0 { + return badRequest("say which figures: keys=zrx:*,ztx:* (docs/stats.md)") + } + tl, err := q.Timeline(r.Context(), d, r.URL.Query().Get("step"), keys) + if err != nil { + return badRequest("%v", err) + } + writeJSON(w, http.StatusOK, tl) + return nil +} + +func (s *Server) statsTop(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + q, err := s.statsQuerier(p) + if err != nil { + return err + } + v, err := since(r, "24h") + if err != nil { + return err + } + d, _ := stats.ParseSince(v) + es, err := q.Top(r.Context(), r.URL.Query().Get("kind"), d, queryInt(r, "limit", 20)) + if err != nil { + return badRequest("%v", err) + } + writeJSON(w, http.StatusOK, es) + return nil +} + +// statsHits are the packets each rule, built-in and policy counted over +// the range, named; rules that counted none are listed too, with 0. +func (s *Server) statsHits(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + q, err := s.statsQuerier(p) + if err != nil { + return err + } + v, err := since(r, "30d") + if err != nil { + return err + } + d, _ := stats.ParseSince(v) + es, err := q.Sums(r.Context(), "hits:", d) + if err != nil { + return err + } + st := s.eng.State() + seen := map[string]bool{} + for i := range es { + es[i].Name = markName(st, es[i].Key) + seen[es[i].Key] = true + } + for _, rule := range st.Rules { + if mark := "r" + strconv.FormatInt(rule.ID, 10); !seen[mark] && !rule.Disabled { + es = append(es, stats.Entry{Key: mark, Name: "rule " + rule.Label()}) + } + } + writeJSON(w, http.StatusOK, es) + return nil +} + +func (s *Server) statsHeatmap(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + q, err := s.statsQuerier(p) + if err != nil { + return err + } + v, err := since(r, "28d") + if err != nil { + return err + } + d, _ := stats.ParseSince(v) + hm, err := q.Heatmap(r.Context(), r.URL.Query().Get("key"), d) + if err != nil { + return err + } + writeJSON(w, http.StatusOK, hm) + return nil +} + +// statsProblems warns when the connection tracking table is nearly full: +// new connections are dropped once it is. +func (s *Server) statsProblems() []validate.Problem { + if s.stats == nil { + return nil + } + l := s.stats.Latest() + if l == nil || l.ConntrackMax == 0 || l.Conntrack*100 < l.ConntrackMax*80 { + return nil + } + return []validate.Problem{{Severity: validate.Warning, Code: "conntrack_full", Object: "connections", + Message: "the connection tracking table is " + strconv.FormatInt(l.Conntrack*100/l.ConntrackMax, 10) + "% full (" + strconv.FormatInt(l.Conntrack, 10) + " of " + strconv.FormatInt(l.ConntrackMax, 10) + "); once it is, new connections are dropped: raise net.netfilter.nf_conntrack_max, or find what opens so many (Connections)"}} +} diff --git a/internal/server/h_status.go b/internal/server/h_status.go index 6294332..0e2b38a 100644 --- a/internal/server/h_status.go +++ b/internal/server/h_status.go @@ -48,6 +48,7 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa out.OtherTables = otherTables(data) } out.Problems = append(out.Problems, s.hostProblems(r.Context(), out.OtherTables)...) + out.Problems = append(out.Problems, s.statsProblems()...) writeJSON(w, http.StatusOK, out) return nil } @@ -296,7 +297,12 @@ func (l *logRing) Latest(n int, after time.Time, keep func(api.LogEntry) bool) [ // LogPacket records a packet the firewall logged (internal/nflog calls // it). -func (s *Server) LogPacket(e api.LogEntry) { s.logs.Add(e) } +func (s *Server) LogPacket(e api.LogEntry) { + s.logs.Add(e) + if s.stats != nil { + s.stats.Packet(e) + } +} func (s *Server) listLog(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { if err := require(p, auth.ResLog, false); err != nil { diff --git a/internal/server/routes.go b/internal/server/routes.go index 254bb61..e407e9c 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -64,6 +64,12 @@ func (s *Server) routes() { m.HandleFunc("POST "+p+"/{kind}/{ref}/move", s.authed(s.moveObject)) m.HandleFunc("GET "+p+"/ruleset", s.authed(s.ruleset)) m.HandleFunc("GET "+p+"/matrix", s.authed(s.zoneMatrix)) + m.HandleFunc("GET "+p+"/stats/status", s.authed(s.statsStatus)) + m.HandleFunc("GET "+p+"/stats/summary", s.authed(s.statsSummary)) + m.HandleFunc("GET "+p+"/stats/timeline", s.authed(s.statsTimeline)) + m.HandleFunc("GET "+p+"/stats/top", s.authed(s.statsTop)) + m.HandleFunc("GET "+p+"/stats/hits", s.authed(s.statsHits)) + m.HandleFunc("GET "+p+"/stats/heatmap", s.authed(s.statsHeatmap)) m.HandleFunc("GET "+p+"/history", s.authed(s.listHistory)) m.HandleFunc("GET "+p+"/history/{id}", s.authed(s.getHistory)) m.HandleFunc("POST "+p+"/history/{id}/revert", s.authed(s.revertHistory)) diff --git a/internal/server/server.go b/internal/server/server.go index c5413fc..987e601 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -24,6 +24,7 @@ import ( "housecarl/internal/model" "housecarl/internal/nft" "housecarl/internal/render" + "housecarl/internal/stats" "housecarl/internal/store" "housecarl/internal/version" ) @@ -97,6 +98,8 @@ type Server struct { conns Conntrack // tracing is the packet trace running (or the last one). tracing tracing + // stats collects statistics (nil: off). + stats *stats.Collector // csMu orders changes to changesets (staging, applying, discarding). csMu sync.Mutex nowFn func() time.Time diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 273915b..aa5044d 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -23,6 +23,7 @@ import ( "housecarl/internal/model" "housecarl/internal/nft" "housecarl/internal/render" + "housecarl/internal/stats" "housecarl/internal/store" ) @@ -994,3 +995,67 @@ func TestTrace(t *testing.T) { t.Errorf("tracing everything: %d", code) } } + +// TestStatsEndpoints: statistics off, then on: the summary's top lists, +// hits per rule (named, with those that counted none), and which +// counters count drops. +func TestStatsEndpoints(t *testing.T) { + e := newEnv(t) + if code := e.call(e.token, "GET", "/stats/summary", nil, nil); code != http.StatusServiceUnavailable { + t.Errorf("statistics off: %d", code) + } + var st StatsStatus + e.call(e.token, "GET", "/stats/status", nil, &st) + if st.Enabled { + t.Error("enabled without a collector") + } + e.call(e.token, "POST", "/zones", model.Zone{Name: "wan", Interfaces: []string{"wan"}, External: true}, nil) + e.call(e.token, "POST", "/rules", model.Rule{Chain: "input", Match: model.Match{Sources: []string{"198.51.100.0/24"}}, Action: "drop", Comment: "bad net"}, nil) + rule := e.eng.State().Rules[0] + + ss, err := stats.OpenStore(":memory:") + if err != nil { + t.Fatal(err) + } + defer ss.Close() + src := stats.Sources{Table: e.fw.Table, SysNet: t.TempDir(), Proc: t.TempDir()} + e.srv.StatsSources(&src) + c := stats.NewCollector(ss, stats.Options{RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour}, src, nil) + e.srv.SetStats(c) + c.Start(context.Background()) + e.srv.LogPacket(api.LogEntry{Time: time.Now(), Source: "198.51.100.7", Protocol: "tcp", DestPort: 22, Verdict: "drop"}) + c.Sample(context.Background()) + + var sum stats.Summary + if code := e.call(e.token, "GET", "/stats/summary?since=1h", nil, &sum); code != http.StatusOK { + t.Fatalf("summary: %d", code) + } + if len(sum.BlockedSources) != 1 || sum.BlockedSources[0].Key != "198.51.100.7" || len(sum.BlockedPorts) != 1 || sum.BlockedPorts[0].Key != "tcp/22" { + t.Errorf("summary: %+v", sum) + } + var hits []stats.Entry + e.call(e.token, "GET", "/stats/hits", nil, &hits) + if len(hits) != 1 || hits[0].Key != "r"+itoa(rule.ID) || hits[0].Count != 0 || !strings.Contains(hits[0].Name, "bad net") { + t.Errorf("hits: %+v", hits) + } + var top []stats.Entry + if code := e.call(e.token, "GET", "/stats/top?kind=blocked_port&since=7d", nil, &top); code != http.StatusOK || len(top) != 1 { + t.Errorf("top: %d %+v", code, top) + } + if code := e.call(e.token, "GET", "/stats/top?kind=nonsense", nil, nil); code != http.StatusBadRequest { + t.Errorf("an unknown list: %d", code) + } + if code := e.call(e.token, "GET", "/stats/timeline?keys=zrx:*,drops&since=2h", nil, nil); code != http.StatusOK { + t.Errorf("timeline: %d", code) + } + e.call(e.token, "GET", "/stats/status", nil, &st) + if !st.Enabled || st.LastSample == nil { + t.Errorf("status: %+v", st) + } + for mark, want := range map[string]string{"r" + itoa(rule.ID): "input true", "p-input": "input true", "p-output": "output false", "b-antispoof6": "early true", "b-established": " false"} { + chain, drop := e.srv.classifyMark(mark) + if got := fmt.Sprint(chain, " ", drop); got != want { + t.Errorf("%s: %q, want %q", mark, got, want) + } + } +} diff --git a/internal/stats/collector.go b/internal/stats/collector.go new file mode 100644 index 0000000..b76b397 --- /dev/null +++ b/internal/stats/collector.go @@ -0,0 +1,546 @@ +package stats + +import ( + "cmp" + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strconv" + "strings" + "sync" + "time" + + "housecarl/internal/api" + "housecarl/internal/nft" +) + +// Resolutions and retention, and the sources the figures come from. +type Options struct { + TopN int + RetainMinute time.Duration + RetainHour time.Duration + RetainDay time.Duration + Location *time.Location // for days (default: local time) + // Interval is how often the collector samples (default: a minute). + Interval time.Duration +} + +// Sources are what the collector reads. Nil functions are left out. +type Sources struct { + // Table reads Housecarl's table: the counters by mark (totals across + // reloads), the quotas, the per-device sets. + Table func(context.Context) (*nft.Table, error) + // Zones maps the interfaces to their zones. + Zones func() map[string]string + // Classify says which chain a mark's counter is in and whether what + // it counts is dropped ("r12" → forward, true). + Classify func(mark string) (chain string, drop bool) + // SysNet and Proc are /sys/class/net and /proc (tests point them + // elsewhere). + SysNet, Proc string +} + +// Counter keys. Traffic: rx:IF, tx:IF (bytes), rxp:IF, txp:IF (packets), +// zrx:ZONE, ztx:ZONE (bytes received from and sent to a zone's +// interfaces). Hits: hits:MARK (packets), hitb:MARK (bytes). Drops: +// drop:CHAIN and drops (packets). Quotas: quota:r12 (bytes). Gauges +// (g:KEY, see gauge) are below. +const ( + GaugeConntrack = "conntrack" + GaugeConntrackMax = "conntrack_max" + GaugeRSS = "rss" // housecarld's resident memory, bytes + GaugeCPU = "cpu" // housecarld's CPU, tenths of a percent of one CPU + GaugeMem = "mem" // the host's memory in use, bytes + GaugeLoad = "load" // the host's 1-minute load average, times 100 +) + +// Top-list kinds: per hour and per day, the TopN largest. +const ( + KindBlockedSource = "blocked_src" // logged drops by source address + KindBlockedPort = "blocked_port" // logged drops by what they tried: tcp/22, icmp + KindDeviceUp = "device_up" // bytes sent to the internet, by inside address + KindDeviceDown = "device_down" // bytes received from it + // A per-address quota's use by address is "quota_r12". +) + +// Kinds are the top lists Top accepts (and quota_rN). +var Kinds = []string{KindBlockedSource, KindBlockedPort, KindDeviceUp, KindDeviceDown} + +// Status describes the collector's health. +type Status struct { + Enabled bool `json:"enabled"` + Database string `json:"database,omitempty"` + Size int64 `json:"size"` + LastSample *time.Time `json:"last_sample,omitempty"` + SampleError string `json:"sample_error,omitempty"` + FirstData *time.Time `json:"first_data,omitempty"` + // PerDevice: the firewall counts traffic per device (a setting). + PerDevice bool `json:"per_device"` + Retention struct { + Minute string `json:"minute"` + Hour string `json:"hour"` + Day string `json:"day"` + } `json:"retention"` +} + +// Latest are the newest gauge values. +type Latest struct { + At time.Time `json:"at"` + Conntrack int64 `json:"conntrack"` + ConntrackMax int64 `json:"conntrack_max"` + RSS int64 `json:"rss"` + CPUPermille int64 `json:"cpu_permille"` + MemUsed int64 `json:"mem_used"` + Load float64 `json:"load"` +} + +// Collector gathers statistics into a Store. +type Collector struct { + opts Options + src Sources + store *Store + log *slog.Logger + now func() time.Time + + // Each source's last totals: the next sample stores the difference. + ifs, table deltas + cpuTicks int64 + cpuAt time.Time + lastPrune time.Time + + mu sync.Mutex + hours map[int64]map[string]map[string]int64 // live hours: hour -> kind -> key -> count + status Status + latest *Latest +} + +// maxKeys bounds each kind of an hour's top list in memory. +const maxKeys = 100_000 + +// NewCollector creates a collector. Run starts it. +func NewCollector(store *Store, opts Options, src Sources, log *slog.Logger) *Collector { + if opts.Location == nil { + opts.Location = time.Local + } + if log == nil { + log = slog.Default() + } + if opts.TopN <= 0 { + opts.TopN = 100 + } + if opts.Interval <= 0 { + opts.Interval = time.Minute + } + if src.SysNet == "" { + src.SysNet = "/sys/class/net" + } + if src.Proc == "" { + src.Proc = "/proc" + } + c := &Collector{opts: opts, src: src, store: store, log: log, now: time.Now, hours: map[int64]map[string]map[string]int64{}} + c.status.Enabled = true + c.status.Database = store.Path() + c.status.Retention.Minute = humanDuration(opts.RetainMinute) + c.status.Retention.Hour = humanDuration(opts.RetainHour) + c.status.Retention.Day = humanDuration(opts.RetainDay) + return c +} + +// SetClock replaces the clock (tests). +func (c *Collector) SetClock(now func() time.Time) { c.now = now } + +// Store returns the database. +func (c *Collector) Store() *Store { return c.store } + +// Options returns the collector's options. +func (c *Collector) Options() Options { return c.opts } + +// Status returns the collector's health. +func (c *Collector) Status() Status { + c.mu.Lock() + defer c.mu.Unlock() + s := c.status + s.Size = c.store.Size() + return s +} + +// Latest returns the newest gauges (nil before the first sample). +func (c *Collector) Latest() *Latest { + c.mu.Lock() + defer c.mu.Unlock() + if c.latest == nil { + return nil + } + l := *c.latest + return &l +} + +func (c *Collector) minuteOf(t time.Time) int64 { return t.Unix() - t.Unix()%60 } +func (c *Collector) hourOf(t time.Time) int64 { return t.Unix() - t.Unix()%3600 } + +// dayOf returns the start of the local day containing t. +func (c *Collector) dayOf(t time.Time) int64 { + t = t.In(c.opts.Location) + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, c.opts.Location).Unix() +} + +// Run collects until ctx is done. +func (c *Collector) Run(ctx context.Context) { + c.Start(ctx) + for { + t := time.NewTimer(time.Until(c.nextSample())) + select { + case <-ctx.Done(): + t.Stop() + c.write(context.Background(), &Batch{}) + return + case <-t.C: + c.Sample(ctx) + } + } +} + +// nextSample: two seconds into the next minute, so the minute that just +// ended is complete (or after the interval, when it isn't a minute). +func (c *Collector) nextSample() time.Time { + if c.opts.Interval != time.Minute { + return c.now().Add(c.opts.Interval) + } + return time.Unix(c.minuteOf(c.now())+62, 0) +} + +// Start takes the first sample (the baseline the next ones are counted +// from), continues the current hour's top lists, and adds hours that +// ended while housecarld wasn't running to their days. +func (c *Collector) Start(ctx context.Context) { + if v, _ := c.store.Meta(ctx, "first_data"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + t := time.Unix(n, 0) + c.status.FirstData = &t + } + } + cur := c.hourOf(c.now()) + b := &Batch{} + merged, _ := c.store.Meta(ctx, "merged_through") + if last, err := strconv.ParseInt(merged, 10, 64); err == nil { + for h := max(last+3600, cur-48*3600); h < cur; h += 3600 { + kinds, _ := c.store.KindsAt(ctx, ResHour, h) + for _, k := range kinds { + if es, err := c.store.TopsAt(ctx, ResHour, h, k); err == nil && len(es) > 0 { + b.AddTops(ResDay, c.dayOf(time.Unix(h, 0)), k, es, 3*c.opts.TopN) + } + } + } + } + b.SetMeta("merged_through", strconv.FormatInt(cur-3600, 10)) + kinds, _ := c.store.KindsAt(ctx, ResHour, cur) + c.mu.Lock() + for _, k := range kinds { + if es, err := c.store.TopsAt(ctx, ResHour, cur, k); err == nil { + m := c.topsMap(cur, k) + for _, e := range es { + m[e.Key] = e.Count + } + } + } + c.mu.Unlock() + if err := c.store.Commit(ctx, b); err != nil { + c.log.Warn("statistics: catching up failed", "err", err) + } + c.sample(ctx, -1) + c.lastPrune = c.now() + c.prune(ctx) + c.log.Info("statistics collector started", "database", c.store.Path()) +} + +// Sample reads every source and writes what changed since the last +// sample. +func (c *Collector) Sample(ctx context.Context) { + now := c.now() + // The interval that just ended: with a minute's interval, sampling two + // seconds into a minute, the previous minute. + minute := c.minuteOf(now.Add(-c.opts.Interval / 2)) + vals := c.sample(ctx, minute) + b := &Batch{} + if len(vals) > 0 { + t := time.Unix(minute, 0) + b.AddCounters(ResMinute, minute, vals) + b.AddCounters(ResHour, c.hourOf(t), vals) + b.AddCounters(ResDay, c.dayOf(t), vals) + c.mu.Lock() + if c.status.FirstData == nil { + c.status.FirstData = &t + b.SetMeta("first_data", strconv.FormatInt(minute, 10)) + } + c.mu.Unlock() + } + c.write(ctx, b) + if now.Sub(c.lastPrune) >= time.Hour { + c.lastPrune = now + c.prune(ctx) + } +} + +// write adds the live hours' top lists to b, ends the hours that are +// over, and commits. +func (c *Collector) write(ctx context.Context, b *Batch) { + c.mu.Lock() + cur := c.hourOf(c.now()) + for h, kinds := range c.hours { + for k, m := range kinds { + b.SetTops(ResHour, h, k, top(m, c.opts.TopN)) + } + if h < cur { + // The hour is over: add it to its day. + for k, m := range kinds { + b.AddTops(ResDay, c.dayOf(time.Unix(h, 0)), k, top(m, c.opts.TopN), 3*c.opts.TopN) + } + delete(c.hours, h) + b.SetMeta("merged_through", strconv.FormatInt(h, 10)) + } + } + c.mu.Unlock() + if err := c.store.Commit(ctx, b); err != nil { + c.log.Error("statistics: writing failed", "err", err) + } +} + +// deltas turn a source's totals into what changed since its last read. +type deltas struct { + prev, next map[string]int64 + based bool +} + +// of returns how much key grew. The first read is the baseline (nothing +// counts); after it, a new key counts whole (a new interface, rule or +// device starts from zero), and a total that went down started again (an +// interface came back, a quota was reset, a device's entry expired). +func (d *deltas) of(key string, cur int64) int64 { + if d.next == nil { + d.next = map[string]int64{} + } + d.next[key] = cur + if !d.based { + return 0 + } + p, ok := d.prev[key] + if !ok || cur < p { + return cur + } + return cur - p +} + +// done keeps this read's totals for the next (a failed read skips it, so +// its keys aren't new next time). +func (d *deltas) done() { d.prev, d.next, d.based = d.next, nil, true } + +// sample reads the sources; minute is the bucket the changes go to (-1: +// only the baseline). It returns the counters. +func (c *Collector) sample(ctx context.Context, minute int64) map[string]int64 { + now := c.now() + vals := map[string]int64{} + add := func(k string, v int64) { + if v != 0 { + vals[k] += v + } + } + gauge := func(k string, v int64) { + vals["g:"+k] += v + vals["gn:"+k]++ + } + tops := map[string]map[string]int64{} + addTop := func(kind, key string, v int64) { + if v <= 0 { + return + } + if tops[kind] == nil { + tops[kind] = map[string]int64{} + } + tops[kind][key] += v + } + var errs []string + + // Interfaces, and their zones. + var zones map[string]string + if c.src.Zones != nil { + zones = c.src.Zones() + } + for name, ic := range readInterfaces(c.src.SysNet) { + rx, tx := c.ifs.of("rx:"+name, ic.RxBytes), c.ifs.of("tx:"+name, ic.TxBytes) + add("rx:"+name, rx) + add("tx:"+name, tx) + add("rxp:"+name, c.ifs.of("rxp:"+name, ic.RxPackets)) + add("txp:"+name, c.ifs.of("txp:"+name, ic.TxPackets)) + if z := zones[name]; z != "" { + add("zrx:"+z, rx) + add("ztx:"+z, tx) + } + } + c.ifs.done() + + // Housecarl's table: hits, drops, quotas, devices. + if c.src.Table != nil { + t, err := c.src.Table(ctx) + switch { + case err == nil: + for mark, ctr := range t.Counters { + p := c.table.of("hits:"+mark, int64(ctr.Packets)) + add("hits:"+mark, p) + add("hitb:"+mark, c.table.of("hitb:"+mark, int64(ctr.Bytes))) + if c.src.Classify != nil && p > 0 { + if chain, drop := c.src.Classify(mark); drop { + add("drop:"+chain, p) + add("drops", p) + } + } + } + for name, used := range t.Quotas { + add("quota:"+strings.TrimPrefix(name, "quota_"), c.table.of("quota:"+name, used)) + } + for set, es := range t.QuotaSets { + kind := strings.TrimSuffix(strings.TrimSuffix(set, "_v4"), "_v6") + for _, e := range es { + addTop(kind, e.Addr, c.table.of(set+"/"+e.Addr, e.Used)) + } + } + for set, es := range t.DeviceSets { + kind := map[bool]string{true: KindDeviceUp, false: KindDeviceDown}[strings.HasPrefix(set, "stats_up")] + for _, e := range es { + key := e.Addr + if strings.HasSuffix(set, "_v6") { + key += "/64" + } + addTop(kind, key, c.table.of(set+"/"+e.Addr, int64(e.Bytes))) + } + } + c.table.done() + case errors.Is(err, nft.ErrNoTable): + default: + errs = append(errs, "reading the table: "+err.Error()) + } + } + + // Gauges. + l := &Latest{At: now} + if n, m, ok := readConntrack(c.src.Proc); ok { + l.Conntrack, l.ConntrackMax = n, m + gauge(GaugeConntrack, n) + gauge(GaugeConntrackMax, m) + } + if rss, ticks, ok := readSelf(c.src.Proc); ok { + l.RSS = rss + gauge(GaugeRSS, rss) + if !c.cpuAt.IsZero() && ticks >= c.cpuTicks { + secs := max(int64(now.Sub(c.cpuAt)/time.Second), 1) + l.CPUPermille = (ticks - c.cpuTicks) * 1000 / clockTicks / secs + gauge(GaugeCPU, l.CPUPermille) + } + c.cpuTicks, c.cpuAt = ticks, now + } + if mem, load, ok := readHost(c.src.Proc); ok { + l.MemUsed, l.Load = mem, load + gauge(GaugeMem, mem) + gauge(GaugeLoad, int64(load*100)) + } + + c.mu.Lock() + defer c.mu.Unlock() + c.latest = l + c.status.LastSample = &now + c.status.SampleError = strings.Join(errs, "; ") + if minute < 0 { + return nil + } + h := c.hourOf(time.Unix(minute, 0)) + for kind, m := range tops { + for k, v := range m { + c.addTop(h, kind, k, v) + } + } + return vals +} + +// Packet counts a packet the firewall logged: drops by source and by +// what they tried. +func (c *Collector) Packet(e api.LogEntry) { + if e.Verdict != "drop" && e.Verdict != "reject" { + return + } + what := e.Protocol + switch { + case e.DestPort > 0: + what = fmt.Sprintf("%s/%d", e.Protocol, e.DestPort) + case e.ICMPType != "": + what = e.Protocol + "/" + e.ICMPType + } + t := e.Time + if t.IsZero() { + t = c.now() + } + h := c.hourOf(t) + c.mu.Lock() + defer c.mu.Unlock() + if e.Source != "" { + c.addTop(h, KindBlockedSource, e.Source, 1) + } + if what != "" { + c.addTop(h, KindBlockedPort, what, 1) + } +} + +// addTop counts into an hour's top list. Called with c.mu held. +func (c *Collector) addTop(hour int64, kind, key string, n int64) { + m := c.topsMap(hour, kind) + if _, ok := m[key]; !ok && len(m) >= maxKeys { + return + } + m[key] += n +} + +func (c *Collector) topsMap(hour int64, kind string) map[string]int64 { + h := c.hours[hour] + if h == nil { + h = map[string]map[string]int64{} + c.hours[hour] = h + } + m := h[kind] + if m == nil { + m = map[string]int64{} + h[kind] = m + } + return m +} + +// top returns the n largest entries of m. +func top(m map[string]int64, n int) []Entry { + out := make([]Entry, 0, len(m)) + for k, v := range m { + out = append(out, Entry{Key: k, Count: v}) + } + slices.SortFunc(out, func(a, b Entry) int { + if a.Count != b.Count { + return cmp.Compare(b.Count, a.Count) + } + return cmp.Compare(a.Key, b.Key) + }) + if len(out) > n { + out = out[:n] + } + return out +} + +func (c *Collector) prune(ctx context.Context) { + now := c.now().Unix() + for res, keep := range map[string]time.Duration{ResMinute: c.opts.RetainMinute, ResHour: c.opts.RetainHour, ResDay: c.opts.RetainDay} { + if keep <= 0 { + continue + } + if n, err := c.store.Prune(ctx, res, now-int64(keep.Seconds())); err != nil { + c.log.Warn("statistics: pruning failed", "err", err) + } else if n > 0 { + c.log.Debug("statistics: pruned", "resolution", res, "rows", n) + } + } +} diff --git a/internal/stats/collector_test.go b/internal/stats/collector_test.go new file mode 100644 index 0000000..4666d3d --- /dev/null +++ b/internal/stats/collector_test.go @@ -0,0 +1,238 @@ +package stats + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "housecarl/internal/api" + "housecarl/internal/nft" + "housecarl/internal/render" +) + +// fakeHost is a /sys/class/net and a /proc to read from. +type fakeHost struct { + t *testing.T + sys, proc string + table *nft.Table + tableError error +} + +func newFakeHost(t *testing.T) *fakeHost { + dir := t.TempDir() + h := &fakeHost{t: t, sys: filepath.Join(dir, "sys"), proc: filepath.Join(dir, "proc"), + table: &nft.Table{Counters: map[string]nft.Counter{}, QuotaUsage: nft.QuotaUsage{Quotas: map[string]int64{}, QuotaSets: map[string][]render.Element{}}, + DeviceSets: map[string][]render.Element{}}} + h.write("proc/sys/net/netfilter/nf_conntrack_count", "120") + h.write("proc/sys/net/netfilter/nf_conntrack_max", "65536") + h.write("proc/self/stat", "4242 (housecarld) S 1 4242 4242 0 -1 4194560 3000 0 0 0 150 50 0 0 20 0 9 0 100 1000000 3000") + h.write("proc/self/status", "Name:\thousecarld\nVmRSS:\t 20480 kB\nThreads:\t9\n") + h.write("proc/meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 6000000 kB\n") + h.write("proc/loadavg", "0.50 0.40 0.30 1/200 4242\n") + return h +} + +func (h *fakeHost) write(rel, content string) { + p := filepath.Join(filepath.Dir(h.sys), rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + h.t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + h.t.Fatal(err) + } +} + +func (h *fakeHost) iface(name string, rx, tx int64) { + h.write("sys/"+name+"/statistics/rx_bytes", fmt.Sprint(rx)) + h.write("sys/"+name+"/statistics/tx_bytes", fmt.Sprint(tx)) + h.write("sys/"+name+"/statistics/rx_packets", fmt.Sprint(rx/100)) + h.write("sys/"+name+"/statistics/tx_packets", fmt.Sprint(tx/100)) +} + +func (h *fakeHost) sources() Sources { + return Sources{ + SysNet: h.sys, Proc: h.proc, + Table: func(context.Context) (*nft.Table, error) { + if h.tableError != nil { + return nil, h.tableError + } + return h.table, nil + }, + Zones: func() map[string]string { return map[string]string{"eth0": "wan", "eth1": "lan"} }, + Classify: func(mark string) (string, bool) { + switch mark { + case "r1": + return "forward", true + case "p-input": + return "input", true + } + return "input", false + }, + } +} + +func TestCollector(t *testing.T) { + ctx := context.Background() + h := newFakeHost(t) + h.iface("eth0", 1000, 500) + h.iface("eth1", 2000, 2000) + h.iface("lo", 9999, 9999) + h.table.Counters["r1"] = nft.Counter{Packets: 10, Bytes: 1000} + h.table.Counters["r2"] = nft.Counter{Packets: 100, Bytes: 9000} + h.table.Quotas["quota_r5"] = 1 << 20 + h.table.DeviceSets["stats_up_v4"] = []render.Element{{Addr: "10.0.10.5", Bytes: 5000}} + + store, err := OpenStore(":memory:") + if err != nil { + t.Fatal(err) + } + defer store.Close() + loc := time.FixedZone("test", 0) + now := time.Date(2026, 10, 19, 10, 0, 30, 0, loc) + c := NewCollector(store, Options{TopN: 10, RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour, Location: loc}, + h.sources(), slog.New(slog.NewTextHandler(io.Discard, nil))) + c.SetClock(func() time.Time { return now }) + c.Start(ctx) + + // A minute of traffic. + h.iface("eth0", 5000, 1500) + h.table.Counters["r1"] = nft.Counter{Packets: 25, Bytes: 2500} + h.table.Counters["p-input"] = nft.Counter{Packets: 3, Bytes: 180} // a new counter counts whole + h.table.Quotas["quota_r5"] = 3 << 20 + h.table.DeviceSets["stats_up_v4"] = []render.Element{{Addr: "10.0.10.5", Bytes: 8000}} + h.table.DeviceSets["stats_down_v6"] = []render.Element{{Addr: "2001:db8:1:2::", Bytes: 70000}} + for range 2 { + c.Packet(api.LogEntry{Time: now, Source: "198.51.100.7", Protocol: "tcp", DestPort: 22, Verdict: "drop"}) + } + c.Packet(api.LogEntry{Time: now, Source: "198.51.100.8", Protocol: "icmp", ICMPType: "echo-request", Verdict: "drop"}) + c.Packet(api.LogEntry{Time: now, Source: "10.0.10.5", Protocol: "udp", DestPort: 53, Verdict: "accept"}) // not a drop + now = time.Date(2026, 10, 19, 10, 1, 2, 0, loc) + c.Sample(ctx) + + q := NewQuerier(c) + sum := func(prefix string) map[string]int64 { + es, err := q.Sums(ctx, prefix, time.Hour) + if err != nil { + t.Fatal(err) + } + m := map[string]int64{} + for _, e := range es { + m[e.Key] = e.Count + } + return m + } + check := func(what string, got, want int64) { + t.Helper() + if got != want { + t.Errorf("%s = %d, want %d", what, got, want) + } + } + check("rx:eth0", sum("rx:")["eth0"], 4000) + check("rx:eth1 (no change)", sum("rx:")["eth1"], 0) + check("rx:lo", sum("rx:")["lo"], 0) + check("zrx:wan", sum("zrx:")["wan"], 4000) + check("ztx:wan", sum("ztx:")["wan"], 1000) + check("hits:r1", sum("hits:")["r1"], 15) + check("hits:p-input", sum("hits:")["p-input"], 3) + check("drop:forward", sum("drop:")["forward"], 15) + check("drop:input", sum("drop:")["input"], 3) + check("quota:r5", sum("quota:")["r5"], 2<<20) + + s, err := q.Summary(ctx, time.Hour, 10) + if err != nil { + t.Fatal(err) + } + check("drops", s.Drops, 18) + if len(s.Zones) == 0 || s.Zones[0].Name != "wan" || s.Zones[0].Received != 4000 { + t.Errorf("zones %+v", s.Zones) + } + if len(s.BlockedSources) != 2 || s.BlockedSources[0] != (Entry{Key: "198.51.100.7", Count: 2}) { + t.Errorf("blocked sources %+v", s.BlockedSources) + } + if len(s.BlockedPorts) != 2 || s.BlockedPorts[0].Key != "tcp/22" || s.BlockedPorts[1].Key != "icmp/echo-request" { + t.Errorf("blocked ports %+v", s.BlockedPorts) + } + if len(s.DevicesUp) != 1 || s.DevicesUp[0] != (Entry{Key: "10.0.10.5", Count: 3000}) { + t.Errorf("devices up %+v", s.DevicesUp) + } + if len(s.DevicesDown) != 1 || s.DevicesDown[0] != (Entry{Key: "2001:db8:1:2::/64", Count: 70000}) { + t.Errorf("devices down %+v", s.DevicesDown) + } + check("peak connections", s.PeakConnections, 120) + + // Gauges average; the CPU needs two samples. + tl, err := q.Timeline(ctx, time.Hour, "1m", []string{"g:" + GaugeRSS, "g:" + GaugeMem, "g:" + GaugeLoad, "rx:*"}) + if err != nil { + t.Fatal(err) + } + last := func(i int) int64 { + for j := len(tl.Times) - 1; j >= 0; j-- { + if tl.Times[j] == time.Date(2026, 10, 19, 10, 0, 0, 0, loc).Unix() { + return tl.Series[i].Values[j] + } + } + return -2 + } + check("rss", last(0), 20480*1024) + check("mem", last(1), 2000000*1024) + check("load", last(2), 50) + if len(tl.Series) != 4 || tl.Series[3].Key != "rx:eth0" { + t.Errorf("rx:* expanded to %+v", tl.Series[3:]) + } + + // Totals that go down started again: an interface came back, a quota + // was reset. A table that can't be read is skipped, and its keys + // aren't new the next time. + h.iface("eth0", 100, 1500) + h.table.Quotas["quota_r5"] = 1000 + h.tableError = fmt.Errorf("nft failed") + now = time.Date(2026, 10, 19, 10, 2, 2, 0, loc) + c.Sample(ctx) + if st := c.Status(); st.SampleError == "" { + t.Error("no sample error") + } + h.tableError = nil + now = time.Date(2026, 10, 19, 10, 3, 2, 0, loc) + c.Sample(ctx) + check("rx:eth0 after a reset", sum("rx:")["eth0"], 4100) + check("quota:r5 after a reset", sum("quota:")["r5"], 2<<20+1000) + check("hits:r1 after a failed read", sum("hits:")["r1"], 15) + + // The hour ends: its top lists go to its day. + now = time.Date(2026, 10, 19, 11, 0, 2, 0, loc) + c.Sample(ctx) + day, err := store.TopsAt(ctx, ResDay, time.Date(2026, 10, 19, 0, 0, 0, 0, loc).Unix(), KindBlockedSource) + if err != nil || len(day) != 2 || day[0].Count != 2 { + t.Errorf("the day's blocked sources: %+v %v", day, err) + } + if es, _ := q.Top(ctx, KindBlockedSource, 30*24*time.Hour, 5); len(es) != 2 { + t.Errorf("30 days of blocked sources: %+v", es) + } + + // A restart continues the hour's top lists. + c.Packet(api.LogEntry{Time: now, Source: "203.0.113.9", Protocol: "tcp", DestPort: 23, Verdict: "drop"}) + c.write(ctx, &Batch{}) + c2 := NewCollector(store, c.opts, h.sources(), c.log) + c2.SetClock(func() time.Time { return now }) + c2.Start(ctx) + c2.Packet(api.LogEntry{Time: now, Source: "203.0.113.9", Protocol: "tcp", DestPort: 23, Verdict: "drop"}) + c2.write(ctx, &Batch{}) + hour, _ := store.TopsAt(ctx, ResHour, time.Date(2026, 10, 19, 11, 0, 0, 0, loc).Unix(), KindBlockedSource) + if len(hour) != 1 || hour[0].Count != 2 { + t.Errorf("after a restart, the hour's blocked sources: %+v", hour) + } + + // The heatmap of drops. + hm, err := q.Heatmap(ctx, "", 7*24*time.Hour) + if err != nil { + t.Fatal(err) + } + if hm.Avg[0][10] != 18 { // a Monday, 10:00 + t.Errorf("heatmap Monday 10:00 = %v", hm.Avg[0][10]) + } +} diff --git a/internal/stats/query.go b/internal/stats/query.go new file mode 100644 index 0000000..97e00ba --- /dev/null +++ b/internal/stats/query.go @@ -0,0 +1,480 @@ +package stats + +import ( + "cmp" + "context" + "fmt" + "slices" + "strconv" + "strings" + "time" +) + +// Querier answers questions about stored statistics. +type Querier struct { + Store *Store + Opts Options + Now func() time.Time +} + +// NewQuerier returns a querier for a collector's data. +func NewQuerier(c *Collector) *Querier { + return &Querier{Store: c.store, Opts: c.opts, Now: c.now} +} + +func (q *Querier) loc() *time.Location { + if q.Opts.Location != nil { + return q.Opts.Location + } + return time.Local +} + +// Resolution picks the finest stored resolution that covers a time range +// starting since ago. +func (q *Querier) Resolution(since time.Duration) string { + switch { + case since <= 48*time.Hour && since <= q.Opts.RetainMinute: + return ResMinute + case since <= q.Opts.RetainHour: + return ResHour + } + return ResDay +} + +// topsResolution picks the resolution for top lists, which exist per hour +// and per day. +func (q *Querier) topsResolution(since time.Duration) string { + if since <= 7*24*time.Hour && since <= q.Opts.RetainHour { + return ResHour + } + return ResDay +} + +// bucketStart returns the start of the bucket containing t at a resolution. +func (q *Querier) bucketStart(res string, t time.Time) int64 { + switch res { + case ResMinute: + return t.Unix() - t.Unix()%60 + case ResHour: + return t.Unix() - t.Unix()%3600 + } + l := t.In(q.loc()) + return time.Date(l.Year(), l.Month(), l.Day(), 0, 0, 0, 0, q.loc()).Unix() +} + +// sums returns the totals of counters (optionally with a key prefix) in +// [from, to). +func (s *Store) sums(ctx context.Context, res string, from, to int64, prefix string) (map[string]int64, error) { + rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(value) FROM counters WHERE res = ? AND ts >= ? AND ts < ? AND key LIKE ? ESCAPE '\' GROUP BY key`, + res, from, to, likePrefix(prefix)) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var k string + var v int64 + if err := rows.Scan(&k, &v); err != nil { + return nil, err + } + out[k] = v + } + return out, rows.Err() +} + +func likePrefix(p string) string { + r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`) + return r.Replace(p) + "%" +} + +// series returns the values of a counter per bucket in [from, to). +func (s *Store) series(ctx context.Context, res string, from, to int64, key string) (map[int64]int64, error) { + rows, err := s.db.QueryContext(ctx, `SELECT ts, value FROM counters WHERE res = ? AND key = ? AND ts >= ? AND ts < ?`, res, key, from, to) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int64]int64{} + for rows.Next() { + var ts, v int64 + if err := rows.Scan(&ts, &v); err != nil { + return nil, err + } + out[ts] = v + } + return out, rows.Err() +} + +// tops sums top lists (of one or more kinds) in [from, to). +func (s *Store) tops(ctx context.Context, res string, kinds []string, from, to int64, limit int) ([]Entry, error) { + args := []any{res} + for _, k := range kinds { + args = append(args, k) + } + args = append(args, from, to, limit) + rows, err := s.db.QueryContext(ctx, `SELECT key, SUM(count) AS n FROM tops WHERE res = ? AND kind IN (`+strings.TrimSuffix(strings.Repeat("?,", len(kinds)), ",")+`) AND ts >= ? AND ts < ? + GROUP BY key ORDER BY n DESC, key LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Entry{} + for rows.Next() { + var e Entry + if err := rows.Scan(&e.Key, &e.Count); err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +func sorted(m map[string]int64, prefix string) []Entry { + out := []Entry{} + for k, v := range m { + if strings.HasPrefix(k, prefix) && v != 0 { + out = append(out, Entry{Key: strings.TrimPrefix(k, prefix), Count: v}) + } + } + slices.SortFunc(out, func(a, b Entry) int { + if a.Count != b.Count { + return cmp.Compare(b.Count, a.Count) + } + return cmp.Compare(a.Key, b.Key) + }) + return out +} + +// Top returns the largest entries of a top list over the last since. +func (q *Querier) Top(ctx context.Context, kind string, since time.Duration, limit int) ([]Entry, error) { + if !slices.Contains(Kinds, kind) && !quotaKind(kind) { + return nil, fmt.Errorf("unknown list %q (use %s, or quota_rN)", kind, strings.Join(Kinds, ", ")) + } + now := q.Now() + res := q.topsResolution(since) + return q.Store.tops(ctx, res, []string{kind}, q.bucketStart(res, now.Add(-since)), now.Unix()+1, limit) +} + +func quotaKind(k string) bool { + n, ok := strings.CutPrefix(k, "quota_r") + _, err := strconv.ParseInt(n, 10, 64) + return ok && err == nil +} + +// Sums returns the totals of the counters with a prefix over the last +// since, largest first, the prefix taken off ("hits:" → "r12"). +func (q *Querier) Sums(ctx context.Context, prefix string, since time.Duration) ([]Entry, error) { + now := q.Now() + res := q.Resolution(since) + m, err := q.Store.sums(ctx, res, q.bucketStart(res, now.Add(-since)), now.Unix()+1, prefix) + if err != nil { + return nil, err + } + return sorted(m, prefix), nil +} + +// Traffic is what an interface or zone received and sent, in bytes. +type Traffic struct { + Name string `json:"name"` + Received int64 `json:"received"` + Sent int64 `json:"sent"` +} + +// Summary is an overview of the last since. +type Summary struct { + Since string `json:"since"` + From time.Time `json:"from"` + Zones []Traffic `json:"zones"` + Interfaces []Traffic `json:"interfaces"` + // Drops are the packets dropped (rules, built-ins, policies), and by + // chain. + Drops int64 `json:"drops"` + DropsByChain []Entry `json:"drops_by_chain"` + // Rules are the busiest rules, built-ins and policies, by packets. + Rules []Entry `json:"rules"` + // Connections: the most tracked at once (per-bucket averages), and + // the table's size. + PeakConnections int64 `json:"peak_connections"` + ConntrackMax int64 `json:"conntrack_max"` + BlockedSources []Entry `json:"blocked_sources"` + BlockedPorts []Entry `json:"blocked_ports"` + DevicesUp []Entry `json:"devices_up"` + DevicesDown []Entry `json:"devices_down"` +} + +// Summary sums up the last since, with top lists of up to topLimit. +func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int) (*Summary, error) { + now := q.Now() + res := q.Resolution(since) + from := q.bucketStart(res, now.Add(-since)) + all, err := q.Store.sums(ctx, res, from, now.Unix()+1, "") + if err != nil { + return nil, err + } + s := &Summary{Since: humanDuration(since), From: time.Unix(from, 0), Zones: []Traffic{}, Interfaces: []Traffic{}} + traffic := func(rx, tx string) []Traffic { + by := map[string]*Traffic{} + for k, v := range all { + var name string + var in bool + if n, ok := strings.CutPrefix(k, rx); ok { + name, in = n, true + } else if n, ok := strings.CutPrefix(k, tx); ok { + name = n + } else { + continue + } + t := by[name] + if t == nil { + t = &Traffic{Name: name} + by[name] = t + } + if in { + t.Received += v + } else { + t.Sent += v + } + } + out := []Traffic{} + for _, t := range by { + out = append(out, *t) + } + slices.SortFunc(out, func(a, b Traffic) int { + if c := cmp.Compare(b.Received+b.Sent, a.Received+a.Sent); c != 0 { + return c + } + return cmp.Compare(a.Name, b.Name) + }) + return out + } + s.Zones, s.Interfaces = traffic("zrx:", "ztx:"), traffic("rx:", "tx:") + s.Drops = all["drops"] + s.DropsByChain = sorted(all, "drop:") + s.Rules = sorted(all, "hits:") + if len(s.Rules) > topLimit { + s.Rules = s.Rules[:topLimit] + } + if tl, err := q.Timeline(ctx, since, "auto", []string{"g:" + GaugeConntrack, "g:" + GaugeConntrackMax}); err == nil { + for _, v := range tl.Series[0].Values { + s.PeakConnections = max(s.PeakConnections, v) + } + for _, v := range tl.Series[1].Values { + s.ConntrackMax = max(s.ConntrackMax, v) + } + } + for _, l := range []struct { + kind string + dst *[]Entry + }{{KindBlockedSource, &s.BlockedSources}, {KindBlockedPort, &s.BlockedPorts}, {KindDeviceUp, &s.DevicesUp}, {KindDeviceDown, &s.DevicesDown}} { + es, err := q.Top(ctx, l.kind, since, topLimit) + if err != nil { + return nil, err + } + *l.dst = es + } + return s, nil +} + +// Timeline is a set of series over time. +type Timeline struct { + Step string `json:"step"` + Times []int64 `json:"times"` // bucket starts (Unix seconds) + Series []*Series `json:"series"` +} + +// Series is one counter over time. For a gauge (g:KEY) the values are +// averages, and -1 where there is no sample. +type Series struct { + Key string `json:"key"` + Values []int64 `json:"values"` +} + +var steps = []struct { + name string + d time.Duration + res string +}{ + {"1m", time.Minute, ResMinute}, {"5m", 5 * time.Minute, ResMinute}, {"15m", 15 * time.Minute, ResMinute}, + {"1h", time.Hour, ResHour}, {"6h", 6 * time.Hour, ResHour}, {"1d", 24 * time.Hour, ResDay}, +} + +// Timeline returns counters over the last since. keys are counter names; +// a key ending in ":*" means every counter of that group. step is "auto" +// or one of 1m, 5m, 15m, 1h, 6h, 1d. +func (q *Querier) Timeline(ctx context.Context, since time.Duration, step string, keys []string) (*Timeline, error) { + idx := -1 + for i, st := range steps { + ok := st.res == ResMinute && since <= q.Opts.RetainMinute || st.res == ResHour && since <= q.Opts.RetainHour || st.res == ResDay + if step == st.name { + if !ok { + return nil, fmt.Errorf("step %s needs %s data, which is only kept for %s", st.name, map[string]string{ResMinute: "per-minute", ResHour: "hourly"}[st.res], humanDuration(map[string]time.Duration{ResMinute: q.Opts.RetainMinute, ResHour: q.Opts.RetainHour}[st.res])) + } + idx = i + break + } + if step == "auto" || step == "" { + if ok && since/st.d <= 300 { + idx = i + break + } + } + } + if idx < 0 { + if step != "auto" && step != "" { + return nil, fmt.Errorf("unknown step %q (use auto, 1m, 5m, 15m, 1h, 6h or 1d)", step) + } + idx = len(steps) - 1 + } + st := steps[idx] + now := q.Now() + // Buckets aligned to the step (days to local midnight). + var starts []int64 + if st.res == ResDay { + for t := time.Unix(q.bucketStart(ResDay, now.Add(-since)), 0).In(q.loc()); !t.After(now); t = t.AddDate(0, 0, 1) { + starts = append(starts, t.Unix()) + } + } else { + sd := int64(st.d.Seconds()) + first := now.Add(-since).Unix() + first -= first % sd + for t := first; t <= now.Unix(); t += sd { + starts = append(starts, t) + } + } + from, to := starts[0], now.Unix()+1 + // Expand group keys. + var expanded []string + for _, k := range keys { + if g, ok := strings.CutSuffix(k, ":*"); ok { + m, err := q.Store.sums(ctx, st.res, from, to, g+":") + if err != nil { + return nil, err + } + for _, e := range sorted(m, "") { + expanded = append(expanded, e.Key) + } + continue + } + expanded = append(expanded, k) + } + tl := &Timeline{Step: st.name, Times: starts, Series: []*Series{}} + bucketed := func(key string) ([]int64, error) { + raw, err := q.Store.series(ctx, st.res, from, to, key) + if err != nil { + return nil, err + } + vals := make([]int64, len(starts)) + for ts, v := range raw { + // The bucket is the last start <= ts. + i, found := slices.BinarySearch(starts, ts) + if !found { + i-- + } + if i >= 0 && i < len(vals) { + vals[i] += v + } + } + return vals, nil + } + for _, k := range expanded { + vals, err := bucketed(k) + if err != nil { + return nil, err + } + // A gauge (g:KEY) is the average of its samples: its sum divided + // by gn:KEY, their number. A bucket without samples is -1. + if g, ok := strings.CutPrefix(k, "g:"); ok { + n, err := bucketed("gn:" + g) + if err != nil { + return nil, err + } + for i := range vals { + if n[i] > 0 { + vals[i] /= n[i] + } else { + vals[i] = -1 + } + } + } + tl.Series = append(tl.Series, &Series{Key: k, Values: vals}) + } + return tl, nil +} + +// Heatmap is average activity per weekday and hour of day (local time). +type Heatmap struct { + Key string `json:"key"` + // Avg[d][h] is the average per hour on weekday d (0 = Monday) at + // hour h; -1 where there is no data. + Avg [7][24]float64 `json:"avg"` + // Samples[d][h] is how many hours the average is based on. + Samples [7][24]int `json:"samples"` + From time.Time `json:"from"` +} + +// Heatmap averages a counter (default "drops") by weekday and hour over +// the last since (at most the hourly retention). +func (q *Querier) Heatmap(ctx context.Context, key string, since time.Duration) (*Heatmap, error) { + if key == "" { + key = "drops" + } + since = min(since, q.Opts.RetainHour) + now := q.Now() + from := q.bucketStart(ResHour, now.Add(-since)) + raw, err := q.Store.series(ctx, ResHour, from, now.Unix()+1, key) + if err != nil { + return nil, err + } + hm := &Heatmap{Key: key, From: time.Unix(from, 0)} + var sum [7][24]int64 + for ts, v := range raw { + t := time.Unix(ts, 0).In(q.loc()) + d := (int(t.Weekday()) + 6) % 7 + sum[d][t.Hour()] += v + hm.Samples[d][t.Hour()]++ + } + for d := range 7 { + for h := range 24 { + if n := hm.Samples[d][h]; n > 0 { + hm.Avg[d][h] = float64(sum[d][h]) / float64(n) + } else { + hm.Avg[d][h] = -1 + } + } + } + return hm, nil +} + +// ParseSince parses a range such as 90m, 24h, 7d or 2w. +func ParseSince(s string) (time.Duration, error) { + if s == "" { + return 24 * time.Hour, nil + } + mult := time.Duration(0) + switch { + case strings.HasSuffix(s, "d"): + mult = 24 * time.Hour + case strings.HasSuffix(s, "w"): + mult = 7 * 24 * time.Hour + } + if mult > 0 { + var n int + if _, err := fmt.Sscanf(s[:len(s)-1], "%d", &n); err != nil || n <= 0 || fmt.Sprint(n) != s[:len(s)-1] { + return 0, fmt.Errorf("invalid range %q (use e.g. 90m, 24h, 7d, 2w)", s) + } + return time.Duration(n) * mult, nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, fmt.Errorf("invalid range %q (use e.g. 90m, 24h, 7d, 2w)", s) + } + return d, nil +} + +func humanDuration(d time.Duration) string { + if d%(24*time.Hour) == 0 && d > 0 { + return strconv.Itoa(int(d/(24*time.Hour))) + "d" + } + return d.String() +} diff --git a/internal/stats/sources.go b/internal/stats/sources.go new file mode 100644 index 0000000..8e31dfd --- /dev/null +++ b/internal/stats/sources.go @@ -0,0 +1,136 @@ +package stats + +import ( + "bytes" + "os" + "path/filepath" + "strconv" + "strings" +) + +// What the collector reads from the kernel without privileges: the +// interfaces' counters (/sys/class/net), the connection tracking table's +// size, and housecarld's and the host's memory and load (/proc). + +// clockTicks is USER_HZ, the unit of the CPU times in /proc/PID/stat. It +// is 100 on every Linux architecture Housecarl runs on. +const clockTicks = 100 + +// ifCounters are an interface's totals since it came up. +type ifCounters struct { + RxBytes, TxBytes, RxPackets, TxPackets int64 +} + +// readInterfaces reads every interface's counters but the loopback's. +func readInterfaces(sysNet string) map[string]ifCounters { + out := map[string]ifCounters{} + entries, err := os.ReadDir(sysNet) + if err != nil { + return out + } + for _, e := range entries { + name := e.Name() + if name == "lo" { + continue + } + dir := filepath.Join(sysNet, name, "statistics") + read := func(f string) (int64, bool) { + b, err := os.ReadFile(filepath.Join(dir, f)) + if err != nil { + return 0, false + } + n, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + return n, err == nil + } + var c ifCounters + var ok bool + if c.RxBytes, ok = read("rx_bytes"); !ok { + continue + } + c.TxBytes, _ = read("tx_bytes") + c.RxPackets, _ = read("rx_packets") + c.TxPackets, _ = read("tx_packets") + out[name] = c + } + return out +} + +// readConntrack returns how many connections the kernel tracks, and how +// many it can (ok false without connection tracking). +func readConntrack(proc string) (count, max int64, ok bool) { + read := func(f string) (int64, bool) { + b, err := os.ReadFile(filepath.Join(proc, "sys/net/netfilter", f)) + if err != nil { + return 0, false + } + n, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + return n, err == nil + } + count, ok1 := read("nf_conntrack_count") + max, ok2 := read("nf_conntrack_max") + return count, max, ok1 && ok2 +} + +// readSelf returns housecarld's resident memory (bytes) and CPU time +// (clock ticks). +func readSelf(proc string) (rss, cpu int64, ok bool) { + dir := filepath.Join(proc, "self") + stat, err := os.ReadFile(filepath.Join(dir, "stat")) + if err != nil { + return 0, 0, false + } + // The command name is in parentheses and may contain spaces: the + // fields that follow start after the last ')'. + i := bytes.LastIndexByte(stat, ')') + if i < 0 { + return 0, 0, false + } + f := strings.Fields(string(stat[i+1:])) + // f[0] is field 3 (state); utime and stime are fields 14 and 15. + if len(f) < 13 { + return 0, 0, false + } + utime, _ := strconv.ParseInt(f[11], 10, 64) + stime, _ := strconv.ParseInt(f[12], 10, 64) + status, err := os.ReadFile(filepath.Join(dir, "status")) + if err != nil { + return 0, 0, false + } + for _, line := range strings.Split(string(status), "\n") { + if k, v, ok := strings.Cut(line, ":"); ok && k == "VmRSS" { + rss = kib(v) + } + } + return rss, utime + stime, true +} + +// readHost returns the memory in use (bytes) and the 1-minute load +// average. +func readHost(proc string) (memUsed int64, load float64, ok bool) { + b, err := os.ReadFile(filepath.Join(proc, "meminfo")) + if err != nil { + return 0, 0, false + } + var total, avail int64 + for _, line := range strings.Split(string(b), "\n") { + k, v, _ := strings.Cut(line, ":") + switch k { + case "MemTotal": + total = kib(v) + case "MemAvailable": + avail = kib(v) + } + } + if b, err := os.ReadFile(filepath.Join(proc, "loadavg")); err == nil { + if f := strings.Fields(string(b)); len(f) > 0 { + load, _ = strconv.ParseFloat(f[0], 64) + } + } + return total - avail, load, total > 0 +} + +// kib parses a "1234 kB" value from /proc into bytes. +func kib(v string) int64 { + n, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(v), "kB")), 10, 64) + return n * 1024 +} diff --git a/internal/stats/store.go b/internal/stats/store.go new file mode 100644 index 0000000..0868dfe --- /dev/null +++ b/internal/stats/store.go @@ -0,0 +1,304 @@ +// Package stats keeps the firewall's figures over time: traffic per +// interface and zone, hits per rule, drops, blocked sources, connections, +// quotas and the machine's health, sampled every minute. It stores +// aggregates in a SQLite database of its own, separate from Housecarl's +// configuration, so it can be deleted at any time without losing anything +// important. +package stats + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + + _ "modernc.org/sqlite" // registers the "sqlite" driver +) + +// Resolutions of stored data. +const ( + ResMinute = "m" + ResHour = "h" + ResDay = "d" +) + +// Store is the statistics database. +type Store struct { + db *sql.DB + path string +} + +var schema = []string{ + `CREATE TABLE counters ( + res TEXT NOT NULL, + ts INTEGER NOT NULL, + key TEXT NOT NULL, + value INTEGER NOT NULL, + PRIMARY KEY (res, ts, key) + ) WITHOUT ROWID; + CREATE INDEX counters_by_key ON counters (res, key, ts); + CREATE TABLE tops ( + res TEXT NOT NULL, + ts INTEGER NOT NULL, + kind TEXT NOT NULL, + key TEXT NOT NULL, + count INTEGER NOT NULL, + PRIMARY KEY (res, ts, kind, key) + ) WITHOUT ROWID; + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);`, +} + +// Size is the database's size on disk, with its write-ahead log. +func (s *Store) Size() int64 { + var n int64 + for _, f := range []string{s.path, s.path + "-wal"} { + if fi, err := os.Stat(f); err == nil { + n += fi.Size() + } + } + return n +} + +// Oldest is the start of the oldest stored bucket (0 when empty). +func (s *Store) Oldest(ctx context.Context) (int64, error) { + var ts sql.NullInt64 + err := s.db.QueryRowContext(ctx, `SELECT MIN(ts) FROM counters`).Scan(&ts) + return ts.Int64, err +} + +// OpenStore opens (creating if needed) the statistics database. Use +// ":memory:" for tests. +func OpenStore(path string) (*Store, error) { + dsn := "file:" + path + "?_pragma=busy_timeout(10000)&_pragma=synchronous(NORMAL)&_txlock=immediate" + if path == ":memory:" { + dsn = "file::memory:?_txlock=immediate" + } else { + dsn += "&_pragma=journal_mode(WAL)" + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + if path == ":memory:" { + db.SetMaxOpenConns(1) + } + s := &Store{db: db, path: path} + if err := s.migrate(); err != nil { + db.Close() + return nil, fmt.Errorf("statistics database %s: %w", path, err) + } + return s, nil +} + +// Path returns the database file ("" in memory). +func (s *Store) Path() string { + if s.path == ":memory:" { + return "" + } + return s.path +} + +// Close closes the database. +func (s *Store) Close() error { return s.db.Close() } + +func (s *Store) migrate() error { + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`); err != nil { + return err + } + var v int + err := s.db.QueryRow(`SELECT version FROM schema_version`).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + if _, err := s.db.Exec(`INSERT INTO schema_version (version) VALUES (0)`); err != nil { + return err + } + } else if err != nil { + return err + } + if v > len(schema) { + return fmt.Errorf("schema version %d is newer than this Housecarl; upgrade Housecarl or delete the statistics database", v) + } + for i := v; i < len(schema); i++ { + tx, err := s.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(schema[i]); err != nil { + tx.Rollback() + return err + } + if _, err := tx.Exec(`UPDATE schema_version SET version = ?`, i+1); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +// Batch collects writes that are committed together. +type Batch struct { + counters []counterRow + tops []topsWrite + meta map[string]string +} + +type counterRow struct { + res string + ts int64 + key string + value int64 +} + +type topsWrite struct { + res, kind string + ts int64 + entries []Entry + replace bool // replace the (res, ts, kind) rows instead of adding + keep int // after adding, keep only the largest entries (0: all) +} + +// Entry is a key and its count. +type Entry struct { + Key string `json:"key"` + Count int64 `json:"count"` + // Name is what the key stands for (a rule's label, a device's host + // name), set by the API. + Name string `json:"name,omitempty"` +} + +// AddCounters adds values to the counters of a bucket. +func (b *Batch) AddCounters(res string, ts int64, values map[string]int64) { + for k, v := range values { + b.counters = append(b.counters, counterRow{res, ts, k, v}) + } +} + +// SetTops replaces a bucket's top list of a kind. +func (b *Batch) SetTops(res string, ts int64, kind string, entries []Entry) { + b.tops = append(b.tops, topsWrite{res: res, ts: ts, kind: kind, entries: entries, replace: true}) +} + +// AddTops adds counts to a bucket's top list and keeps the keep largest. +func (b *Batch) AddTops(res string, ts int64, kind string, entries []Entry, keep int) { + b.tops = append(b.tops, topsWrite{res: res, ts: ts, kind: kind, entries: entries, keep: keep}) +} + +// SetMeta stores a value that survives restarts. +func (b *Batch) SetMeta(key, value string) { + if b.meta == nil { + b.meta = map[string]string{} + } + b.meta[key] = value +} + +// Commit writes the batch in one transaction. +func (s *Store) Commit(ctx context.Context, b *Batch) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if len(b.counters) > 0 { + st, err := tx.PrepareContext(ctx, `INSERT INTO counters (res, ts, key, value) VALUES (?, ?, ?, ?) + ON CONFLICT (res, ts, key) DO UPDATE SET value = value + excluded.value`) + if err != nil { + return err + } + for _, c := range b.counters { + if _, err := st.ExecContext(ctx, c.res, c.ts, c.key, c.value); err != nil { + st.Close() + return err + } + } + st.Close() + } + for _, t := range b.tops { + if t.replace { + if _, err := tx.ExecContext(ctx, `DELETE FROM tops WHERE res = ? AND ts = ? AND kind = ?`, t.res, t.ts, t.kind); err != nil { + return err + } + } + for _, e := range t.entries { + if _, err := tx.ExecContext(ctx, `INSERT INTO tops (res, ts, kind, key, count) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (res, ts, kind, key) DO UPDATE SET count = count + excluded.count`, t.res, t.ts, t.kind, e.Key, e.Count); err != nil { + return err + } + } + if t.keep > 0 { + if _, err := tx.ExecContext(ctx, `DELETE FROM tops WHERE res = ? AND ts = ? AND kind = ? AND key NOT IN ( + SELECT key FROM tops WHERE res = ? AND ts = ? AND kind = ? ORDER BY count DESC, key LIMIT ?)`, + t.res, t.ts, t.kind, t.res, t.ts, t.kind, t.keep); err != nil { + return err + } + } + } + for k, v := range b.meta { + if _, err := tx.ExecContext(ctx, `INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, k, v); err != nil { + return err + } + } + return tx.Commit() +} + +// Meta returns a stored value ("" if unset). +func (s *Store) Meta(ctx context.Context, key string) (string, error) { + var v string + err := s.db.QueryRowContext(ctx, `SELECT value FROM meta WHERE key = ?`, key).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return v, err +} + +// Prune deletes data of a resolution older than before (a Unix time). +func (s *Store) Prune(ctx context.Context, res string, before int64) (int64, error) { + var n int64 + for _, table := range []string{"counters", "tops"} { + r, err := s.db.ExecContext(ctx, `DELETE FROM `+table+` WHERE res = ? AND ts < ?`, res, before) + if err != nil { + return n, err + } + c, _ := r.RowsAffected() + n += c + } + return n, nil +} + +// TopsAt returns a bucket's top list of a kind. +func (s *Store) TopsAt(ctx context.Context, res string, ts int64, kind string) ([]Entry, error) { + rows, err := s.db.QueryContext(ctx, `SELECT key, count FROM tops WHERE res = ? AND ts = ? AND kind = ? ORDER BY count DESC, key`, res, ts, kind) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Entry + for rows.Next() { + var e Entry + if err := rows.Scan(&e.Key, &e.Count); err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +// KindsAt returns the top-list kinds stored for one bucket. +func (s *Store) KindsAt(ctx context.Context, res string, ts int64) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT kind FROM tops WHERE res = ? AND ts = ?`, res, ts) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + return nil, err + } + out = append(out, k) + } + return out, rows.Err() +} diff --git a/internal/validate/validate.go b/internal/validate/validate.go index e966071..2d07271 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -185,6 +185,12 @@ func (c *checker) settings() { } } } + if s.StatsPerDevice && len(c.st.ExternalInterfaces()) == 0 { + c.add(Warning, "stats_no_external", "settings", "traffic per device counts what goes to and comes from external zones, and no zone is external (with interfaces)") + } + if s.StatsPerDevice && len(s.FlowtableDevices) > 0 { + c.add(Warning, "stats_flowtable", "settings", "the flowtable's fast path skips the forward chain once a connection is established, so traffic per device only counts each connection's first packets; turn the flowtable off for full figures") + } if p := s.ProbationSeconds; p != 0 && (p < 10 || p > 3600) { c.add(Error, "probation_range", "settings", "the probation is %d seconds; use 10 to 3600", p) } @@ -227,6 +233,11 @@ func (c *checker) sets() { for _, s := range c.st.Sets { obj := "set " + s.Name c.name("set", obj, s.Name, seen) + for _, pre := range []string{"meter_", "conn_", "quota_", "stats_"} { + if strings.HasPrefix(strings.ToLower(s.Name), pre) { + c.add(Error, "set_name_reserved", obj, "set names starting with %s are Housecarl's own (rate limits, quotas, statistics); choose another name", pre) + } + } for _, e := range s.Entries { if _, err := model.ParseAddress(e); err != nil { c.add(Error, "set_entry", obj, "%v", err) diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 40dc3ec..30e1102 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -65,6 +65,15 @@ func TestCodes(t *testing.T) { }}, {"set_entry", func(st *model.State) { st.Sets[0].Entries = []string{"10.0.10.5/24"} }}, {"set_empty", func(st *model.State) { st.Sets[0].Entries = nil }}, + {"set_name_reserved", func(st *model.State) { st.Sets[0].Name = "stats_up" }}, + {"stats_flowtable", func(st *model.State) { + st.Settings.StatsPerDevice = true + st.Settings.FlowtableDevices = []string{"wan", "eth1"} + }}, + {"stats_no_external", func(st *model.State) { + st.Settings.StatsPerDevice = true + st.Zones[0].External = false + }}, {"set_timeout", func(st *model.State) { st.Sets[0].Timeout = 60 }}, {"set_dynamic_entries", func(st *model.State) { st.Sets[0].Dynamic = true }}, {"service_empty", func(st *model.State) { st.Services[0].Entries = nil }}, diff --git a/internal/web/static/assets/app.js b/internal/web/static/assets/app.js index 7889b22..d6cd8c2 100644 --- a/internal/web/static/assets/app.js +++ b/internal/web/static/assets/app.js @@ -6,6 +6,7 @@ import * as changes from "./pages/changes.js"; import * as connections from "./pages/connections.js"; import * as tracePage from "./pages/trace.js"; import * as matrixPage from "./pages/matrix.js"; +import * as statsPage from "./pages/stats.js"; import * as dashboard from "./pages/dashboard.js"; import * as rules from "./pages/rules.js"; import * as nat from "./pages/nat.js"; @@ -32,6 +33,7 @@ const routes = [ { pattern: /^log$/, page: log, nav: "log" }, { pattern: /^connections$/, page: connections, nav: "connections" }, { pattern: /^trace$/, page: tracePage, nav: "trace" }, + { pattern: /^stats$/, page: statsPage, nav: "stats" }, { pattern: /^settings$/, page: settings, nav: "settings" }, { pattern: /^import$/, page: importPage, nav: "import" }, { pattern: /^changes$/, page: changes, nav: "changes" }, @@ -55,6 +57,7 @@ const navItems = [ ["services", "#/services", "Services", "config"], ] }, { menu: "Traffic", items: [ + ["stats", "#/stats", "Statistics", "log"], ["log", "#/log", "Log", "log"], ["connections", "#/connections", "Connections", "log"], ["trace", "#/trace", "Trace", "log"], diff --git a/internal/web/static/assets/charts.js b/internal/web/static/assets/charts.js new file mode 100644 index 0000000..d9afabb --- /dev/null +++ b/internal/web/static/assets/charts.js @@ -0,0 +1,351 @@ +// Small SVG charts for the statistics page: a line/area chart with a +// crosshair tooltip, a weekday x hour heatmap, and bar lists. No library; +// colors come from CSS custom properties (--series-N, --heat-N) so light +// and dark themes are handled in style.css. All text is inserted as text, +// never as markup. +import { h } from "./ui.js"; + +const NS = "http://www.w3.org/2000/svg"; + +function svg(tag, attrs, ...children) { + const el = document.createElementNS(NS, tag); + for (const [k, v] of Object.entries(attrs || {})) { + if (v !== undefined && v !== null && v !== false) el.setAttribute(k, v); + } + for (const c of children.flat(Infinity)) { + if (c === null || c === undefined || c === false) continue; + el.append(c instanceof Node ? c : document.createTextNode(String(c))); + } + return el; +} + +// compact formats a count: 1,234 / 12.9K / 4.2M. +export function compact(n) { + const a = Math.abs(n); + if (a >= 1e9) return (n / 1e9).toFixed(a >= 1e10 ? 0 : 1).replace(/\.0$/, "") + "B"; + if (a >= 1e6) return (n / 1e6).toFixed(a >= 1e7 ? 0 : 1).replace(/\.0$/, "") + "M"; + if (a >= 1e4) return (n / 1e3).toFixed(a >= 1e5 ? 0 : 1).replace(/\.0$/, "") + "K"; + return Math.round(n).toLocaleString(); +} + +// niceTicks returns 0..max in clean steps (1, 2 or 5 x 10^k), at most n+1 ticks. +export function niceTicks(max, n = 4) { + if (max <= 0) return [0, 1]; + const raw = max / n; + const mag = Math.pow(10, Math.floor(Math.log10(raw))); + const step = [1, 2, 5, 10].map((m) => m * mag).find((s) => s >= raw); + const ticks = []; + for (let v = 0; v < max + step * 0.999; v += step) ticks.push(Math.round(v * 1e6) / 1e6); + return ticks; +} + +const MIN = 60, HOUR = 3600, DAY = 86400; +const tickSteps = [5 * MIN, 10 * MIN, 15 * MIN, 30 * MIN, HOUR, 2 * HOUR, 3 * HOUR, 6 * HOUR, 12 * HOUR, DAY, 2 * DAY, 7 * DAY, 14 * DAY, 30 * DAY]; + +// timeTicks picks local-time-aligned ticks between t0 and t1 (Unix seconds). +function timeTicks(t0, t1, maxTicks) { + const span = t1 - t0; + const step = tickSteps.find((s) => span / s <= maxTicks) || 30 * DAY; + const out = []; + const d = new Date(t0 * 1000); + if (step >= DAY) { + d.setHours(0, 0, 0, 0); + } else { + d.setSeconds(0, 0); + const mins = d.getHours() * 60 + d.getMinutes(); + const stepMin = step / 60; + d.setHours(0, Math.floor(mins / stepMin) * stepMin, 0, 0); + } + for (let t = d.getTime() / 1000; t <= t1; ) { + if (t >= t0) out.push(t); + if (step >= DAY) { + const nd = new Date(t * 1000); + nd.setDate(nd.getDate() + step / DAY); + t = nd.getTime() / 1000; + } else t += step; + } + return { ticks: out, step }; +} + +const weekdays = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; +const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; +const pad = (n) => String(n).padStart(2, "0"); + +function tickLabel(t, step) { + const d = new Date(t * 1000); + if (step >= DAY) return step >= 7 * DAY ? months[d.getMonth()] + " " + d.getDate() : weekdays[d.getDay()] + " " + d.getDate(); + const hm = pad(d.getHours()) + ":" + pad(d.getMinutes()); + return d.getHours() === 0 && d.getMinutes() === 0 ? weekdays[d.getDay()] + " " + d.getDate() : hm; +} + +// longTime describes a bucket for tooltips and tables. +export function bucketLabel(t, stepSecs) { + const d = new Date(t * 1000); + const day = weekdays[d.getDay()] + " " + d.getDate() + " " + months[d.getMonth()]; + if (stepSecs >= DAY) return day; + const end = new Date((t + stepSecs) * 1000); + const hm = (x) => pad(x.getHours()) + ":" + pad(x.getMinutes()); + return day + ", " + hm(d) + (stepSecs > MIN ? "–" + hm(end) : ""); +} + +export const stepSeconds = { "1m": MIN, "5m": 5 * MIN, "15m": 15 * MIN, "1h": HOUR, "6h": 6 * HOUR, "1d": DAY }; + +// tooltip is one floating readout per chart. +function tooltip() { + const el = h("div", { class: "chart-tip", role: "status" }); + el.hidden = true; + return el; +} + +// chartCard wraps a chart with its title and a table-view toggle (the +// accessible twin of every chart). +export function chartCard(title, subtitle, chart, tableView, opts = {}) { + const body = h("div", { class: "chart-body" }, chart); + const tableBox = h("div", { class: "chart-table" }, tableView); + tableBox.hidden = true; + const toggle = h("button", { class: "small link", type: "button", "aria-pressed": "false" }, "Table"); + toggle.addEventListener("click", () => { + const showTable = tableBox.hidden; + tableBox.hidden = !showTable; + body.hidden = showTable; + toggle.textContent = showTable ? "Chart" : "Table"; + toggle.setAttribute("aria-pressed", String(showTable)); + }); + return h("figure", { class: "panel chart-card" + (opts.wide ? " wide" : "") }, + h("figcaption", { class: "chart-head" }, + h("div", { class: "grow" }, h("h2", null, title), subtitle ? h("div", { class: "muted small" }, subtitle) : null), + opts.legend || null, + tableView ? toggle : null), + body, tableBox); +} + +// legend lists series with a short line key (the mark the series uses). +export function legend(series) { + return h("ul", { class: "chart-legend" }, series.map((s) => + h("li", null, h("span", { class: "key line s" + s.slot }), s.label))); +} + +// endLabelWidth is the room lineChart needs for the end labels of series. +export function endLabelWidth(series) { + return series.length > 1 && series.length <= 4 ? Math.max(...series.map((s) => s.label.length)) * 7 + 12 : 8; +} + +// lineChart draws series over time. series: [{label, values, slot}] (slot +// = categorical color 1..3); times: bucket starts (Unix seconds); step in +// seconds. With area, the first series gets a 10% wash under its line. +// gutter {left, right} sets the least room for the axis labels and the end +// labels: charts shown one above the other pass the same gutter, so their +// time axes line up. format formats values (axis and readout); a null +// value is a gap in the line (no data for that time). +export function lineChart({ series, times, step, area = false, height = 220, endLabels = true, gutter = {}, format = compact }) { + const root = h("div", { class: "chart", tabindex: "0", "aria-label": series.map((s) => s.label).join(", ") + " over time" }); + const tip = tooltip(); + root.append(tip); + let width = 0, focusIdx = -1, geom = null; + + function draw() { + const w = root.clientWidth; + if (!w || w === width) return; + width = w; + root.querySelector("svg")?.remove(); + const max = Math.max(1, ...series.flatMap((s) => s.values).filter((v) => v != null)); + const yTicks = niceTicks(max, 4); + const yMax = yTicks[yTicks.length - 1]; + const labelW = Math.max(...yTicks.map((t) => format(t).length)) * 7 + 10; + const endW = endLabels ? endLabelWidth(series) : 8; + const m = { top: 10, right: Math.max(endW, gutter.right || 0), bottom: 24, left: Math.max(labelW, gutter.left || 0) }; + const pw = Math.max(40, w - m.left - m.right), ph = height - m.top - m.bottom; + const t0 = times[0], t1 = times[times.length - 1] + step; + const x = (t) => m.left + ((t - t0) / Math.max(1, t1 - t0)) * pw; + const cx = (i) => x(times[i] + step / 2); // the middle of a bucket + const y = (v) => m.top + ph - (v / yMax) * ph; + const s = svg("svg", { width: w, height, viewBox: `0 0 ${w} ${height}`, role: "img", "aria-hidden": "true" }); + // Grid and axes. + for (const t of yTicks) { + s.append(svg("line", { class: t === 0 ? "axis" : "grid", x1: m.left, x2: m.left + pw, y1: y(t), y2: y(t) })); + s.append(svg("text", { class: "tick", x: m.left - 6, y: y(t) + 4, "text-anchor": "end" }, format(t))); + } + const { ticks, step: tstep } = timeTicks(t0, t1, Math.max(2, Math.floor(pw / 90))); + for (const t of ticks) { + s.append(svg("text", { class: "tick", x: x(t), y: height - 6, "text-anchor": "middle" }, tickLabel(t, tstep))); + } + // Series. + series.forEach((sr, k) => { + // A gap (null) starts a new piece of line. + const pts = sr.values.map((v, i) => (v == null ? null : [cx(i), y(v)])); + const d = pts.map((p, i) => (p ? (i && pts[i - 1] ? "L" : "M") + p[0].toFixed(1) + " " + p[1].toFixed(1) : "")).join(""); + if (area && k === 0 && pts.length && pts.every(Boolean)) { + s.append(svg("path", { class: "area s" + sr.slot, d: d + `L${pts[pts.length - 1][0].toFixed(1)} ${y(0)}L${pts[0][0].toFixed(1)} ${y(0)}Z` })); + } + s.append(svg("path", { class: "line s" + sr.slot, d })); + // A point without neighbours has no line: show it as a dot. + pts.forEach((p, i) => { + if (p && !pts[i - 1] && !pts[i + 1]) s.append(svg("circle", { class: "dot s" + sr.slot, cx: p[0], cy: p[1], r: 3 })); + }); + }); + // Direct end labels when there is room and they don't collide. + if (m.right > 8) { + const ends = series.map((sr) => ({ label: sr.label, y: y(sr.values.findLast((v) => v != null) || 0) })).sort((a, b) => a.y - b.y); + if (ends.every((e, i) => i === 0 || e.y - ends[i - 1].y >= 13)) { + for (const e of ends) s.append(svg("text", { class: "end-label", x: m.left + pw + 6, y: e.y + 4 }, e.label)); + } + } + // Crosshair (drawn on hover/focus). + const cross = svg("line", { class: "crosshair", y1: m.top, y2: m.top + ph, visibility: "hidden" }); + const dots = series.map((sr) => svg("circle", { class: "dot s" + sr.slot, r: 4, visibility: "hidden" })); + s.append(cross, ...dots); + // The hit area is the whole plot: readers aim at a time, not a line. + const hit = svg("rect", { class: "hit", x: m.left, y: m.top, width: pw, height: ph }); + s.append(hit); + root.prepend(s); + geom = { cx, y, m, pw, ph, cross, dots, hit }; + hit.addEventListener("pointermove", (e) => { + const r = s.getBoundingClientRect(); + const px = e.clientX - r.left; + const i = Math.round(((px - m.left) / pw) * (times.length) - 0.5); + show(Math.max(0, Math.min(times.length - 1, i))); + }); + hit.addEventListener("pointerleave", hide); + if (focusIdx >= 0) show(focusIdx); + } + + function show(i) { + if (!geom) return; + focusIdx = i; + const xpos = geom.cx(i); + geom.cross.setAttribute("x1", xpos); + geom.cross.setAttribute("x2", xpos); + geom.cross.setAttribute("visibility", "visible"); + series.forEach((sr, k) => { + geom.dots[k].setAttribute("cx", xpos); + geom.dots[k].setAttribute("cy", geom.y(sr.values[i] ?? 0)); + geom.dots[k].setAttribute("visibility", sr.values[i] == null ? "hidden" : "visible"); + }); + tip.replaceChildren( + h("div", { class: "tip-head" }, bucketLabel(times[i], step)), + ...series.map((sr) => h("div", { class: "tip-row" }, + h("span", { class: "key line s" + sr.slot }), h("strong", null, sr.values[i] == null ? "–" : format === compact ? sr.values[i].toLocaleString() : format(sr.values[i])), " ", h("span", { class: "muted" }, sr.label)))); + tip.hidden = false; + // Keep the readout inside the chart, on the far side of the pointer. + const left = xpos > root.clientWidth / 2 ? xpos - tip.offsetWidth - 12 : xpos + 12; + tip.style.left = Math.max(0, left) + "px"; + tip.style.top = geom.m.top + "px"; + } + + function hide() { + if (!geom) return; + geom.cross.setAttribute("visibility", "hidden"); + geom.dots.forEach((d) => d.setAttribute("visibility", "hidden")); + tip.hidden = true; + focusIdx = -1; + } + + // Keyboard: arrows move the crosshair, as hover does. + root.addEventListener("focus", () => show(focusIdx >= 0 ? focusIdx : times.length - 1)); + root.addEventListener("blur", hide); + root.addEventListener("keydown", (e) => { + if (e.key === "ArrowLeft" || e.key === "ArrowRight") { + e.preventDefault(); + show(Math.max(0, Math.min(times.length - 1, (focusIdx < 0 ? times.length - 1 : focusIdx) + (e.key === "ArrowLeft" ? -1 : 1)))); + } + }); + new ResizeObserver(draw).observe(root); + return root; +} + +// timeTable is the table view of a line chart (busiest buckets first is +// not useful here: keep time order, newest first). +export function timeTable(series, times, step, format) { + const cell = (v) => (v == null ? "–" : format ? format(v) : v.toLocaleString()); + const rows = times.map((t, i) => ({ t, i })).reverse(); + return h("div", { class: "table-wrap scroll" }, h("table", null, + h("thead", null, h("tr", null, h("th", null, "Time"), series.map((s) => h("th", { class: "num" }, s.label)))), + h("tbody", null, rows.map((r) => h("tr", null, h("td", null, bucketLabel(r.t, step)), + series.map((s) => h("td", { class: "num" }, cell(s.values[r.i])))))))); +} + +// heatmap draws average activity by weekday (rows, Monday first) and hour +// (columns) with a sequential one-hue ramp; cells without data stay empty. +export function heatmap(hm, unit) { + const days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; + let peak = 0; + hm.avg.forEach((row) => row.forEach((v) => { peak = Math.max(peak, v); })); + const steps = 7; + const level = (v) => (peak <= 0 ? 0 : Math.min(steps - 1, Math.floor((v / peak) * steps))); + const tip = tooltip(); + const grid = h("div", { class: "heatmap", role: "grid", "aria-label": "Average " + unit + " per hour by weekday and hour of day" }); + grid.append(h("div", { class: "hm-corner" })); + for (let hr = 0; hr < 24; hr++) grid.append(h("div", { class: "hm-col" }, hr % 3 === 0 ? pad(hr) : "")); + days.forEach((day, d) => { + grid.append(h("div", { class: "hm-row" }, day)); + for (let hr = 0; hr < 24; hr++) { + const v = hm.avg[d][hr]; + const n = hm.samples[d][hr]; + const text = v < 0 ? day + " " + pad(hr) + ":00 – no data" : + day + " " + pad(hr) + ":00 – " + Math.round(v).toLocaleString() + " " + unit + " per hour (average of " + n + (n === 1 ? " hour)" : " hours)"); + const cell = h("div", { class: "hm-cell " + (v < 0 ? "hm-empty" : "heat-" + level(v)), tabindex: "-1", "aria-label": text }); + const showTip = () => { + tip.textContent = text; + tip.hidden = false; + const r = cell.getBoundingClientRect(), pr = grid.getBoundingClientRect(); + tip.style.left = Math.max(0, Math.min(pr.width - tip.offsetWidth, r.left - pr.left - tip.offsetWidth / 2 + r.width / 2)) + "px"; + tip.style.top = (r.bottom - pr.top + 6) + "px"; + }; + cell.addEventListener("pointerenter", showTip); + cell.addEventListener("focus", showTip); + cell.addEventListener("pointerleave", () => { tip.hidden = true; }); + cell.addEventListener("blur", () => { tip.hidden = true; }); + grid.append(cell); + } + }); + // Arrow keys move between cells. + grid.tabIndex = 0; + let cur = -1; + const cells = () => [...grid.querySelectorAll(".hm-cell")]; + grid.addEventListener("keydown", (e) => { + const delta = { ArrowLeft: -1, ArrowRight: 1, ArrowUp: -24, ArrowDown: 24 }[e.key]; + if (delta === undefined) return; + e.preventDefault(); + cur = Math.max(0, Math.min(7 * 24 - 1, (cur < 0 ? 0 : cur) + delta)); + cells()[cur].focus(); + }); + const scale = h("div", { class: "hm-scale" }, + h("span", { class: "muted small" }, "0"), + Array.from({ length: steps }, (_, i) => h("span", { class: "hm-swatch heat-" + i })), + h("span", { class: "muted small" }, compact(peak) + " " + unit + "/hour"), + h("span", { class: "hm-swatch hm-empty" }), h("span", { class: "muted small" }, "no data")); + return h("div", { class: "heatmap-wrap" }, grid, tip, scale); +} + +export function heatmapTable(hm, unit) { + const days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]; + return h("div", { class: "table-wrap scroll" }, h("table", null, + h("thead", null, h("tr", null, h("th", null, "Hour"), days.map((d) => h("th", { class: "num" }, d)))), + h("tbody", null, Array.from({ length: 24 }, (_, hr) => h("tr", null, h("td", null, pad(hr) + ":00"), + days.map((_, d) => h("td", { class: "num" }, hm.avg[d][hr] < 0 ? "–" : Math.round(hm.avg[d][hr]).toLocaleString())))))), + h("div", { class: "muted small" }, "Average " + unit + " per hour.")); +} + +// barList shows ranked entries as labelled horizontal bars in one hue +// (one series: no legend, the title names it). Values are always printed, +// so the list is its own table view. +// extra(key), when given, adds something after an entry's label (e.g. a +// link that excludes it). +export function barList(entries, { total, label = (k) => k, empty = "No data yet.", href, extra, format = compact } = {}) { + if (!entries.length) return h("div", { class: "empty" }, empty); + // Entries may be in a natural order (e.g. latency buckets), not by size. + const peak = Math.max(1, ...entries.map((e) => e.count)); + const sum = total || entries.reduce((a, e) => a + e.count, 0); + return h("ol", { class: "bar-list" }, entries.map((e) => { + const share = sum ? (e.count / sum) * 100 : 0; + const name = label(e.key); + const link = href ? href(e.key) : null; + // Entries with a name (a rule's label): the name, then the key. + const shown = e.name ? [e.name, h("span", { class: "muted" }, " " + name)] : name; + return h("li", { title: (e.name ? e.name + " (" + name + ")" : name) + ": " + format(e.count) + " (" + share.toFixed(1) + "%)" }, + h("div", { class: "bar-text" }, + link ? h("a", { href: link, class: "bar-label" }, shown) : h("span", { class: "bar-label" }, shown), + h("span", { class: "bar-value" }, extra ? [extra(e.key), " "] : null, format(e.count), h("span", { class: "muted" }, " · " + share.toFixed(share < 10 ? 1 : 0) + "%"))), + h("div", { class: "bar-track" }, h("div", { class: "bar-fill", style: { width: Math.max(0.5, (e.count / peak) * 100) + "%" } }))); + })); +} diff --git a/internal/web/static/assets/pages/dashboard.js b/internal/web/static/assets/pages/dashboard.js index bc7291f..373a26a 100644 --- a/internal/web/static/assets/pages/dashboard.js +++ b/internal/web/static/assets/pages/dashboard.js @@ -4,15 +4,19 @@ import { h, mount, table, badge, problemsList, time } from "../ui.js"; import { get, can } from "../api.js"; import { onLeave } from "../app.js"; import { describe, hits, humanCount } from "../widgets.js"; +import { lineChart, chartCard, legend, timeTable, stepSeconds } from "../charts.js"; +import { bytes } from "./stats.js"; export async function render(el) { const draw = async () => { const st = await get("/status"); const config = can("config", false), logs = can("log", false); - const [rules, cs, log] = await Promise.all([ + const [rules, cs, log, tl, zones] = await Promise.all([ config ? get("/rules") : [], logs ? get("/counters").then((c) => c.counters).catch(() => ({})) : {}, logs ? get("/log", { limit: 8 }).catch(() => []) : [], + logs ? get("/stats/timeline", { since: "24h", step: "1h", keys: "zrx:*,ztx:*,drops" }).catch(() => null) : null, + config ? get("/zones").catch(() => []) : [], ]); const busiest = rules.filter((r) => cs["r" + r.id] && cs["r" + r.id].packets > 0) .sort((a, b) => cs["r" + b.id].packets - cs["r" + a.id].packets).slice(0, 8); @@ -35,6 +39,7 @@ export async function render(el) { card("Dropped by policy", policyDrops("input") + " / " + policyDrops("forward"), "input / forward", logs ? "#/log" : null), card("Problems", st.problems.length, st.problems.length ? "see below" : "none", null)), st.problems.length ? h("div", { class: "panel" }, h("h2", null, "Problems"), problemsList(st.problems)) : null, + tl ? statsCharts(tl, zones) : null, h("div", { class: "panel" }, h("h2", null, "Interfaces"), table([ { label: "Interface", render: (i) => [h("span", { class: "dot " + (i.up ? "ok" : "err") }), h("span", { class: "mono" }, h("strong", null, i.name))] }, { label: "Zone", render: (i) => i.zone || h("span", { class: "muted" }, "none") }, @@ -63,3 +68,20 @@ export async function render(el) { const t = setInterval(() => { if (!document.hidden) draw().catch(() => {}); }, 15000); onLeave(() => clearInterval(t)); } + +// statsCharts: the last 24 hours' traffic with the internet and drops, +// per hour (the Statistics page has the rest). +function statsCharts(tl, zones) { + const ext = new Set(zones.filter((z) => z.external).map((z) => z.name)); + const step = stepSeconds[tl.step]; + const total = (prefix) => tl.times.map((_, i) => tl.series.filter((x) => x.key.startsWith(prefix) && ext.has(x.key.slice(prefix.length))).reduce((a, x) => a + x.values[i], 0)); + const traffic = [{ label: "Downloaded", values: total("zrx:"), slot: 1 }, { label: "Uploaded", values: total("ztx:"), slot: 2 }]; + const drops = [{ label: "Dropped", values: (tl.series.find((x) => x.key === "drops") || { values: tl.times.map(() => 0) }).values, slot: 2 }]; + const gutter = { left: 64, right: 8 }; + return h("div", { class: "chart-grid" }, + ext.size ? chartCard("Internet traffic", "bytes per hour from and to external zones, last 24 hours", + lineChart({ series: traffic, times: tl.times, step, area: true, height: 150, endLabels: false, gutter, format: bytes }), + timeTable(traffic, tl.times, step, bytes), { legend: legend(traffic) }) : null, + chartCard("Dropped packets", h("span", null, "per hour, last 24 hours · ", h("a", { href: "#/stats" }, "Statistics")), + lineChart({ series: drops, times: tl.times, step, area: true, height: 150, endLabels: false, gutter }), timeTable(drops, tl.times, step))); +} diff --git a/internal/web/static/assets/pages/settings.js b/internal/web/static/assets/pages/settings.js index 37fe453..8f486f8 100644 --- a/internal/web/static/assets/pages/settings.js +++ b/internal/web/static/assets/pages/settings.js @@ -27,6 +27,7 @@ export async function render(el) { const v6 = listEditor(s.antispoof_v6, { addLabel: "Add" }); const flow = listEditor(s.flowtable_devices, { suggestions: ifs.map((i) => i.name), placeholder: "wan", addLabel: "Add device" }); const probation = input({ value: fmtSeconds(s.probation_seconds || 120), class: "short" }); + const perDevice = checkbox("count traffic per inside device (IPv6 per /64), for the Statistics page's top devices: a counter for each address talking to an external zone", s.stats_per_device); const manual = checkbox("a person confirms each change (otherwise the browser or hc that made it confirms it by itself, once it reaches Housecarl again over a new connection)", s.manual_confirm); const err = h("div"); const save = async (dry) => { @@ -38,7 +39,7 @@ export async function render(el) { try { const r = await change("PATCH", "/settings", { input_policy: input_.value, forward_policy: forward.value, output_policy: output.value, log_policy_drops: logDrops.input.checked, builtins, own_prefixes: own.value(), antispoof_v4: v4.value(), antispoof_v6: v6.value(), - flowtable_devices: flow.value(), probation_seconds: p === 120 ? 0 : p, manual_confirm: manual.input.checked }, { query: dry ? { dry_run: 1 } : undefined }); + flowtable_devices: flow.value(), probation_seconds: p === 120 ? 0 : p, manual_confirm: manual.input.checked, stats_per_device: perDevice.input.checked }, { query: dry ? { dry_run: 1 } : undefined }); if (!dry && (r.applied || r.staged)) rerender(); } catch (e) { mount(err, errorBox(e)); } }; @@ -56,7 +57,8 @@ export async function render(el) { field("Antispoof IPv6", v6), field("Flowtable", flow, "Forwarded TCP and UDP between these devices skip the rules once established (fast path). Exact names; none: off."), field("Probation", probation, "How long a change made over the network has to be confirmed before it is rolled back."), - h("label"), manual), + h("label"), manual, + field("Statistics", perDevice)), err, write ? h("div", { class: "actions form-actions" }, h("button", { class: "primary", onclick: () => save(false) }, "Save"), diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js new file mode 100644 index 0000000..66ca70e --- /dev/null +++ b/internal/web/static/assets/pages/stats.js @@ -0,0 +1,214 @@ +// Statistics: traffic per zone, drops, blocked sources, devices, +// connections, quotas and the machine over time (docs/stats.md). +import { h, mount, fmtTime, errorBox } from "../ui.js"; +import { get } from "../api.js"; +import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js"; + +const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"], ["365d", "a year"]]; +const rangeSecs = { "1h": 3600, "6h": 6 * 3600, "24h": 86400, "7d": 7 * 86400, "30d": 30 * 86400, "90d": 90 * 86400, "365d": 365 * 86400 }; + +let generation = 0; +// The zone the traffic chart shows, kept while the page refreshes. +let shownZone = null; + +export async function render(el, params, query) { + const gen = ++generation; + const want = query && query.get ? query.get("since") : null; + const since = rangeSecs[want] ? want : "24h"; + const status = await get("/stats/status"); + const head = h("div", { class: "page-head" }, + h("div", { class: "grow" }, h("h1", null, "Statistics"), + h("div", { class: "muted" }, status.enabled + ? "Sampled every minute · data since " + (status.first_data ? fmtTime(status.first_data) : "–") + + " · kept: minutes " + status.retention.minute + ", hours " + status.retention.hour + ", days " + status.retention.day + : "Statistics are off"))); + if (!status.enabled) { + mount(el, head, h("div", { class: "panel" }, + h("p", null, "Housecarl keeps traffic per zone, drops, blocked sources, connections and quotas over time, in a database of its own. Turn it on in /etc/housecarl/housecarl.yaml and restart housecarl:"), + h("pre", { class: "code" }, "stats:\n enabled: true"), + h("p", { class: "muted" }, "See docs/stats.md."))); + return; + } + const heatSince = rangeSecs[since] < 7 * 86400 ? "7d" : rangeSecs[since] > 90 * 86400 ? "90d" : since; + let d; + try { + d = await loadAll(since, heatSince); + } catch (e) { + mount(el, head, rangeRow(since), errorBox(e)); + return; + } + if (gen !== generation) return; // the user moved on + const s = d.summary; + const rangeLabel = "in the last " + ranges.find((r) => r[0] === since)[1]; + const parts = [head, rangeRow(since)]; + if (status.sample_error) parts.push(h("div", { class: "msg warn" }, "The last sample failed: " + status.sample_error)); + + // Headline numbers: what came from and went to the external zones. + const ext = new Set(d.zones.filter((z) => z.external).map((z) => z.name)); + const sum = (f) => s.zones.filter((z) => ext.has(z.name)).reduce((a, z) => a + f(z), 0); + const l = status.latest || {}; + parts.push(h("div", { class: "cards" }, + ext.size ? tile("Downloaded", bytes(sum((z) => z.received)), "from external zones, " + rangeLabel) : null, + ext.size ? tile("Uploaded", bytes(sum((z) => z.sent)), "to external zones") : null, + tile("Dropped", compact(s.drops), "packets: rules, built-ins and policies"), + s.conntrack_max ? tile("Connections", compact(s.peak_connections), "most at once (of " + compact(s.conntrack_max) + " allowed)" + (l.conntrack != null ? " · now " + compact(l.conntrack) : "")) : null)); + + // Traffic per zone: one zone at a time, received and sent. + const traffic = complete(d.traffic); + const step = stepSeconds[traffic.step]; + const zoneNames = s.zones.map((z) => z.name); + if (!zoneNames.includes(shownZone)) shownZone = zoneNames.find((z) => ext.has(z)) || zoneNames[0] || null; + const trafficBox = h("div"); + const drawTraffic = () => { + const pick = h("div", { class: "segmented" }, zoneNames.map((z) => + h("button", { type: "button", class: z === shownZone ? "active" : "", onclick: () => { shownZone = z; drawTraffic(); } }, z))); + const series = [ + { label: "Received", values: values(traffic, "zrx:" + shownZone), slot: 1 }, + { label: "Sent", values: values(traffic, "ztx:" + shownZone), slot: 2 }, + ]; + mount(trafficBox, chartCard("Traffic" + (shownZone ? ": " + shownZone : ""), + "bytes per " + stepName(traffic.step) + " received from and sent to the zone's interfaces, " + rangeLabel, + shownZone ? h("div", null, zoneNames.length > 1 ? h("div", { class: "filter-row" }, pick) : null, + lineChart({ series, times: traffic.times, step, area: true, gutter: { left: 64, right: endLabelWidth(series) }, format: bytes })) + : h("div", { class: "empty" }, "No traffic recorded yet: interfaces count once they are in a zone."), + shownZone ? timeTable(series, traffic.times, step, bytes) : null, { wide: true, legend: shownZone ? legend(series) : null })); + }; + drawTraffic(); + parts.push(trafficBox); + if (s.zones.length) { + parts.push(h("div", { class: "chart-grid" }, + listCard("Received per zone", rangeLabel, barList(s.zones.map((z) => ({ key: z.name, count: z.received })), { format: bytes })), + listCard("Sent per zone", rangeLabel, barList(s.zones.map((z) => ({ key: z.name, count: z.sent })), { format: bytes })))); + } + + // Drops over time, by chain, and when they happen. + const drops = complete(d.drops); + const dSeries = [{ label: "Dropped", values: values(drops, "drops"), slot: 2 }]; + const anyDrops = dSeries[0].values.some((v) => v > 0); + parts.push(chartCard("Dropped packets", "per " + stepName(drops.step) + ", by rules, built-ins and policies that drop, " + rangeLabel, + anyDrops ? lineChart({ series: dSeries, times: drops.times, step: stepSeconds[drops.step], area: true, gutter: { left: 64, right: 8 } }) : h("div", { class: "empty" }, "Nothing dropped " + rangeLabel + "."), + anyDrops ? timeTable(dSeries, drops.times, stepSeconds[drops.step]) : null, { wide: true })); + parts.push(chartCard("When drops happen", "average dropped packets per hour by weekday and hour of day (local time), last " + ranges.find((r) => r[0] === heatSince)[1], + heatmap(d.heat, "packets"), heatmapTable(d.heat, "packets"), { wide: true })); + + // Who and what. + const unused = d.hits.filter((e) => e.count === 0 && e.key.startsWith("r")); + const devHint = h("div", { class: "empty" }, status.per_device + ? "No traffic per device " + rangeLabel + "." + : ["Traffic per device is off. Turn it on under ", h("a", { href: "#/settings" }, "Settings"), " (it adds a counter for each inside address)."]); + parts.push(h("div", { class: "chart-grid" }, + listCard("Busiest rules", "packets each counted, " + rangeLabel, barList(s.rules, { label: (k) => k, href: (k) => ruleHref(k) })), + listCard("Dropped, by chain", "early: antispoofing · invalid: packets conntrack can't place", barList(s.drops_by_chain, { empty: "Nothing dropped " + rangeLabel + "." })), + listCard("Blocked most", "sources of logged drops (the policies' with 'log drops' on, and rules that log)", barList(s.blocked_sources, { empty: "No logged drops " + rangeLabel + ".", href: (k) => "#/trace?address=" + encodeURIComponent(k) })), + listCard("What they tried", "the logged drops' protocol and port", barList(s.blocked_ports, { empty: "No logged drops " + rangeLabel + "." })), + listCard("Devices, received", "bytes from external zones, by inside address (IPv6 per /64)", s.devices_down.length ? barList(s.devices_down, { format: bytes, href: connHref }) : devHint), + listCard("Devices, sent", "bytes to external zones", s.devices_up.length ? barList(s.devices_up, { format: bytes, href: connHref }) : devHint.cloneNode(true)), + d.quotas.length ? listCard("Quotas", "bytes counted against each rule's quota, " + rangeLabel, barList(d.quotas, { format: bytes, label: (k) => "rule #" + k.slice(1), href: () => "#/rules" })) : null, + listCard("Rules that matched nothing", "in the last 30 days (or since statistics started): candidates to remove", + unused.length ? h("div", null, h("ul", { class: "plain-list" }, unused.slice(0, 12).map((e) => h("li", null, h("a", { href: ruleHref(e.key) }, e.name || e.key)))), + unused.length > 12 ? h("div", { class: "muted small" }, "and " + (unused.length - 12) + " more: hc stats rules --unused") : null) : h("div", { class: "empty" }, "Every rule matched something.")))); + + // Connections and the machine. + const g = complete(d.gauges); + const gStep = stepSeconds[g.step]; + const gauge = (k, f = (v) => v) => values(g, k).map((v) => (v == null || v < 0 ? null : f(v))); + const any = (ss) => ss.some((x) => x.values.some((v) => v != null)); + const cSeries = [{ label: "Tracked", values: gauge("g:conntrack"), slot: 1 }]; + const mSeries = [{ label: "Host in use", values: gauge("g:mem"), slot: 1 }, { label: "housecarld", values: gauge("g:rss"), slot: 2 }]; + const lSeries = [{ label: "Load", values: gauge("g:load", (v) => v / 100), slot: 1 }, { label: "housecarld CPU %", values: gauge("g:cpu", (v) => v / 10), slot: 2 }]; + const gutter = { left: 64, right: Math.max(endLabelWidth(cSeries), endLabelWidth(mSeries), endLabelWidth(lSeries)) }; + const empty = h("div", { class: "empty" }, "No samples yet: they are taken once a minute."); + parts.push(h("h2", { class: "section" }, "Connections and the machine")); + parts.push(chartCard("Tracked connections", "average per " + stepName(g.step) + (l.conntrack_max ? "; the kernel allows " + compact(l.conntrack_max) + " (nf_conntrack_max), and drops new connections beyond" : ""), + any(cSeries) ? lineChart({ series: cSeries, times: g.times, step: gStep, area: true, gutter }) : empty, any(cSeries) ? timeTable(cSeries, g.times, gStep) : null, { wide: true })); + parts.push(chartCard("Memory", "average per " + stepName(g.step) + (l.rss ? " · housecarld now " + bytes(l.rss) : ""), + any(mSeries) ? lineChart({ series: mSeries, times: g.times, step: gStep, gutter, format: bytes }) : empty.cloneNode(true), any(mSeries) ? timeTable(mSeries, g.times, gStep, bytes) : null, { wide: true, legend: legend(mSeries) })); + parts.push(chartCard("Load", "the host's 1-minute load average, and housecarld's CPU (percent of one CPU)", + any(lSeries) ? lineChart({ series: lSeries, times: g.times, step: gStep, gutter, format: (v) => (Math.round(v * 100) / 100).toString() }) : empty.cloneNode(true), + any(lSeries) ? timeTable(lSeries, g.times, gStep, (v) => v.toFixed(2)) : null, { wide: true, legend: legend(lSeries) })); + el.classList.remove("refreshing"); + mount(el, parts); + scheduleRefresh(el, params, query, gen); +} + +async function loadAll(since, heatSince) { + const [summary, zones, traffic, drops, heat, hits, gauges, quotas] = await Promise.all([ + get("/stats/summary", { since, limit: 10 }), + get("/zones"), + get("/stats/timeline", { since, keys: "zrx:*,ztx:*" }), + get("/stats/timeline", { since, keys: "drops" }), + get("/stats/heatmap", { since: heatSince }), + get("/stats/hits", { since: "30d" }), + get("/stats/timeline", { since, keys: "g:conntrack,g:conntrack_max,g:mem,g:rss,g:load,g:cpu" }), + get("/stats/timeline", { since, keys: "quota:*" }), + ]); + return { summary, zones, traffic, drops, heat, hits, gauges, + quotas: quotas.series.map((x) => ({ key: x.key.slice(6), count: x.values.reduce((a, v) => a + Math.max(v, 0), 0) })).filter((e) => e.count > 0).sort((a, b) => b.count - a.count) }; +} + +function scheduleRefresh(el, params, query, gen) { + setTimeout(() => { + const hh = location.hash; + if (gen !== generation || !(hh === "#/stats" || hh.startsWith("#/stats?")) || !el.isConnected) return; + el.classList.add("refreshing"); + render(el, params, query).catch(() => el.classList.remove("refreshing")); + }, 60_000); +} + +// complete drops the last bucket of a timeline if it hasn't ended yet: +// drawn, it would look like a sudden fall. +function complete(tl) { + const step = stepSeconds[tl.step]; + const n = tl.times.length; + if (n > 1 && tl.times[n - 1] + step > Date.now() / 1000) { + return { ...tl, times: tl.times.slice(0, -1), series: tl.series.map((x) => ({ ...x, values: x.values.slice(0, -1) })) }; + } + return tl; +} + +function values(tl, key) { + return (tl.series.find((x) => x.key === key) || { values: tl.times.map(() => 0) }).values; +} + +function rangeRow(since) { + return h("div", { class: "filter-row", role: "group", "aria-label": "Time range" }, + h("span", { class: "muted" }, "Range"), + h("div", { class: "segmented" }, ranges.map(([k, label]) => + h("a", { href: "#/stats?since=" + k, class: k === since ? "active" : "", "aria-current": k === since ? "true" : null, title: "last " + label }, k === "365d" ? "1y" : k)))); +} + +// ruleHref: a counter's mark to its page ("r12" → the rules, "b-…" → the +// built-ins under Settings). +function ruleHref(mark) { + if (/^r\d/.test(mark)) return "#/rules"; + if (mark.startsWith("b-") || mark.startsWith("p-")) return "#/settings"; + if (mark.startsWith("f")) return "#/nat"; + return null; +} + +function connHref(addr) { + return "#/connections?q=" + encodeURIComponent(addr.replace(/::\/64$/, ":")); +} + +// bytes formats a size in binary units. +export function bytes(n) { + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let i = 0; + while (n >= 1024 && i < units.length - 1) { + n /= 1024; + i++; + } + return (i === 0 || n >= 100 ? Math.round(n) : n.toFixed(1).replace(/\.0$/, "")) + " " + units[i]; +} + +function tile(label, value, sub) { + return h("div", { class: "card stat" }, h("div", { class: "label" }, label), h("div", { class: "value big" }, value), h("div", { class: "sub" }, sub)); +} + +function listCard(title, subtitle, body) { + return h("section", { class: "panel" }, h("h2", null, title), subtitle ? h("div", { class: "muted small list-sub" }, subtitle) : null, body); +} + +function stepName(step) { + return { "1m": "minute", "5m": "5 minutes", "15m": "15 minutes", "1h": "hour", "6h": "6 hours", "1d": "day" }[step] || step; +} diff --git a/internal/web/static/assets/pages/trace.js b/internal/web/static/assets/pages/trace.js index 9163236..055b35b 100644 --- a/internal/web/static/assets/pages/trace.js +++ b/internal/web/static/assets/pages/trace.js @@ -4,8 +4,8 @@ import { h, mount, input, select, field, errorBox, badge } from "../ui.js"; import { get, api, can } from "../api.js"; import { onLeave } from "../app.js"; -export async function render(el) { - const address = input({ placeholder: "192.168.1.50, or 2001:db8:1::/64", class: "mono" }); +export async function render(el, params, query) { + const address = input({ placeholder: "192.168.1.50, or 2001:db8:1::/64", class: "mono", value: (query && query.get && query.get("address")) || "" }); const port = input({ placeholder: "443", class: "short", type: "number", min: 1, max: 65535 }); const protocol = select([["", "any"], "tcp", "udp", "sctp", "icmp", "icmpv6"], "", { class: "inline" }); const seconds = input({ value: "60", class: "short", type: "number", min: 5, max: 600 }); diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index 789f25d..f3d9f2b 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -430,3 +430,136 @@ td.matrix-blocked { background: var(--del); color: var(--del-text); } .matrix-n { margin-left: 6px; font-weight: 400; font-size: 12px; color: var(--warn); } td.matrix-none { color: var(--muted); } tr.matrix-rest td { border-top: 2px solid var(--border); } + +/* ---- Statistics charts. Colors from the validated reference palette + (categorical slots 1-3, sequential blue), checked against this UI's + panel surfaces in both themes. */ +:root { + --series-1: #2a78d6; + --series-2: #eb6834; + --series-3: #1baf7a; + --chart-grid: #eceef2; + --chart-axis: #c9cfd8; + /* Sequential ramp, low to high (blue 100 -> 700). */ + --heat-0: #cde2fb; --heat-1: #9ec5f4; --heat-2: #6da7ec; --heat-3: #3987e5; + --heat-4: #256abf; --heat-5: #184f95; --heat-6: #0d366b; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --series-1: #3987e5; + --series-2: #d95926; + --series-3: #199e70; + --chart-grid: #272c34; + --chart-axis: #3a414d; + /* In dark, low values recede into the surface: the ramp runs dark -> light. */ + --heat-0: #0d366b; --heat-1: #184f95; --heat-2: #256abf; --heat-3: #3987e5; + --heat-4: #6da7ec; --heat-5: #9ec5f4; --heat-6: #cde2fb; + } +} +:root[data-theme="dark"] { + --series-1: #3987e5; + --series-2: #d95926; + --series-3: #199e70; + --chart-grid: #272c34; + --chart-axis: #3a414d; + --heat-0: #0d366b; --heat-1: #184f95; --heat-2: #256abf; --heat-3: #3987e5; + --heat-4: #6da7ec; --heat-5: #9ec5f4; --heat-6: #cde2fb; +} + +.refreshing { opacity: 0.6; transition: opacity 0.2s; } +/* Secondary row actions show when the row is hovered or focused; touch + screens, which can't hover, always show them. Opacity keeps them in the + layout (no shifting) and reachable with Tab. */ +tr .reveal { opacity: 0; transition: opacity 0.1s; } +tr:hover .reveal, tr:focus-within .reveal { opacity: 1; } +@media (hover: none) { tr .reveal { opacity: 1; } } +/* Blocking steps, in order, on the Blocking page. */ +.steps-line { display: flex; flex-wrap: wrap; align-items: center; gap: 4px; margin: 4px 0 10px; } +.step-chip { display: inline-flex; align-items: center; gap: 4px; padding: 2px 8px; border: 1px solid var(--border); border-radius: 12px; font-family: var(--mono); font-size: 13px; background: var(--panel); } +.step-chip.accent { border-color: var(--accent); } +.step-tools { display: inline-flex; gap: 2px; margin-left: 2px; } +.step-tools button { padding: 0 5px; min-width: 0; } +h2.section { margin: 24px 0 10px; font-size: 18px; } +.segmented button { border: none; border-right: 1px solid var(--border); border-radius: 0; background: transparent; padding: 4px 12px; color: var(--text); } +.segmented button:last-child { border-right: none; } +.segmented button:hover { background: var(--info-soft); } +.segmented button.active { background: var(--accent); color: var(--accent-text); font-weight: 600; } +.card.stat .value.big { font-size: 26px; font-weight: 600; line-height: 1.2; } + +.chart-card { margin: 0 0 14px; } +.chart-head { display: flex; align-items: flex-start; gap: 12px; flex-wrap: wrap; margin-bottom: 8px; } +.chart-head .grow { flex: 1; min-width: 200px; } +.chart-head h2 { margin: 0; font-size: 16px; } +/* Two or three columns; the last cards stretch to fill their row, so no + card is left alone beside an empty space. */ +.chart-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 14px; } +.chart-grid > :last-child:nth-child(odd) { grid-column: 1 / -1; } +@media (min-width: 1100px) { + .chart-grid { grid-template-columns: repeat(3, minmax(0, 1fr)); } + .chart-grid > :last-child:nth-child(odd) { grid-column: auto; } + .chart-grid > :last-child:nth-child(3n+1) { grid-column: 1 / -1; } + .chart-grid > :last-child:nth-child(3n+2) { grid-column: span 2; } +} +.chart-grid > .panel h2 { margin: 0; font-size: 16px; } +.list-sub { margin: 2px 0 8px; } + +.chart { position: relative; outline: none; } +.chart:focus-visible { box-shadow: 0 0 0 2px var(--accent); border-radius: var(--radius); } +.chart svg { display: block; overflow: visible; } +.chart .grid { stroke: var(--chart-grid); stroke-width: 1; } +.chart .axis { stroke: var(--chart-axis); stroke-width: 1; } +.chart .tick, .chart .end-label { fill: var(--muted); font-size: 11px; font-variant-numeric: tabular-nums; } +.chart .end-label { fill: var(--text); } +.chart .line { fill: none; stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart .area { stroke: none; fill-opacity: 0.1; } +.chart .dot { stroke: var(--panel); stroke-width: 2; } +.chart .crosshair { stroke: var(--muted); stroke-width: 1; } +.chart .hit { fill: transparent; cursor: crosshair; } +.chart .s1, .key.s1 { stroke: var(--series-1); } +.chart .s2, .key.s2 { stroke: var(--series-2); } +.chart .s3, .key.s3 { stroke: var(--series-3); } +.chart .area.s1, .chart .dot.s1 { fill: var(--series-1); } +.chart .area.s2, .chart .dot.s2 { fill: var(--series-2); } +.chart .area.s3, .chart .dot.s3 { fill: var(--series-3); } +.key.line { display: inline-block; width: 14px; height: 0; border-top: 2px solid; vertical-align: middle; margin-right: 6px; } +.key.line.s1 { border-color: var(--series-1); } +.key.line.s2 { border-color: var(--series-2); } +.key.line.s3 { border-color: var(--series-3); } +.chart-legend { list-style: none; display: flex; gap: 14px; margin: 2px 0 0; padding: 0; font-size: 13px; color: var(--text); flex-wrap: wrap; } + +.chart-tip { position: absolute; z-index: 5; pointer-events: none; background: var(--panel); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.16); padding: 6px 10px; font-size: 13px; white-space: nowrap; } +.chart-tip .tip-head { color: var(--muted); font-size: 12px; margin-bottom: 2px; } +.chart-tip .tip-row strong { font-variant-numeric: tabular-nums; } +.chart-table .scroll, .chart-table .table-wrap { max-height: 360px; overflow: auto; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } + +.heatmap-wrap { position: relative; } +.heatmap { display: grid; grid-template-columns: 36px repeat(24, minmax(10px, 1fr)); gap: 2px; outline: none; } +.heatmap:focus-visible { box-shadow: 0 0 0 2px var(--accent); } +.hm-corner, .hm-col, .hm-row { font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; } +.hm-row { display: flex; align-items: center; } +.hm-cell { height: 22px; border-radius: 3px; } +.hm-cell:hover, .hm-cell:focus { outline: 2px solid var(--text); outline-offset: -1px; } +.hm-empty { background: transparent; box-shadow: inset 0 0 0 1px var(--chart-grid); } +.heat-0 { background: var(--heat-0); } .heat-1 { background: var(--heat-1); } .heat-2 { background: var(--heat-2); } +.heat-3 { background: var(--heat-3); } .heat-4 { background: var(--heat-4); } .heat-5 { background: var(--heat-5); } +.heat-6 { background: var(--heat-6); } +.hm-scale { display: flex; align-items: center; gap: 2px; margin-top: 10px; flex-wrap: wrap; } +.hm-scale .muted { margin: 0 6px; } +.hm-swatch { display: inline-block; width: 18px; height: 10px; border-radius: 2px; } + +.bar-list { list-style: none; margin: 0; padding: 0; } +.bar-list li { padding: 4px 0; } +.bar-text { display: flex; gap: 8px; justify-content: space-between; font-size: 13px; } +.bar-label { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-family: var(--mono); font-size: 12.5px; min-width: 0; } +.bar-value { white-space: nowrap; font-variant-numeric: tabular-nums; } +.bar-track { height: 6px; margin-top: 3px; } +.bar-fill { height: 6px; background: var(--series-1); border-radius: 0 3px 3px 0; min-width: 2px; } +@media (max-width: 700px) { + .heatmap { grid-template-columns: 30px repeat(24, minmax(6px, 1fr)); gap: 1px; } + .hm-cell { height: 16px; } + .chart-grid { grid-template-columns: 1fr; } +} +.plain-list { margin: 0; padding-left: 18px; } +.plain-list li { padding: 2px 0; } diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 21e8d1c..09ad826 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -100,8 +100,9 @@ database: /w/data/housecarl.db ruleset: /w/data/ruleset.nft backup: {dir: /w/backups} log: {level: debug} +stats: {database: /w/data/stats.db} EOF -start() { R sh -c '(/w/housecarld -c /w/housecarl.yaml serve >> /w/srv.log 2>&1 &); for i in $(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1'; } +start() { R sh -c '(HOUSECARL_STATS_INTERVAL=3s /w/housecarld -c /w/housecarl.yaml serve >> /w/srv.log 2>&1 &); for i in $(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1'; } # ---- today's router: the hand-written ruleset @@ -284,6 +285,33 @@ run "the internet to the firewall" 0 H matrix wan firewall contains "blocked *sources that can't come from outside" contains "allowed *udp/51820" +# ---- statistics (sampled every 3 seconds here) + +run "traffic per device on" 0 H settings set --stats-per-device on --log-drops on +run "the device counters are in the ruleset" 0 R nft list set inet housecarl stats_up_v4 +send 500000 >/dev/null 2>&1 +run "a connection from the internet, dropped and logged" 1 connect net $WAN4 23 +sleep 8 +run "statistics are collected" 0 H stats status +contains "Collecting: *yes" +contains "Per device: *on" +run "the summary" 0 H stats --since 1h +contains "^wan *[0-9.]* [KM]B" +contains "Dropped: *[1-9]" +run "traffic per zone over time" 0 H stats traffic --since 1h --step 1m +contains "wan ↓" +run "the devices" 0 H stats top devices --since 1h +contains "^$CLIENT4 *[0-9.]* KB *[0-9.]* KB" +run "the sources blocked" 0 H stats top blocked --since 1h +contains "^$NET4 *[1-9]" +run "what they tried" 0 H stats top tried --since 1h +contains "^tcp/23" +run "hits per rule" 0 H stats rules --since 1h +contains "input policy *[1-9]" +contains "rule #[0-9]* (deleted) *[1-9]" +run "connections over time" 0 H stats connections --since 1h --step 1m +contains "^[0-9-]* [0-9:]* *[0-9]* *[0-9]*" + # ---- connections and tracing # A connection that stays open, masqueraded on its way out.