abrown38andClaude Opus 5.5
ca5ac971e5
The established built-in passes most of the traffic, so it topped the statistics' busiest rules every time and made every rule's bar look empty. The summary now lists rules and NAT objects in rules, and built-ins and policies in builtins; the Statistics page and hc stats show them as two lists. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
420 lines
13 KiB
Go
420 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// Statistics: the figures housecarld keeps over time (docs/stats.md). The
|
|
// types mirror internal/stats's JSON, so hc doesn't link the database.
|
|
|
|
type statEntry struct {
|
|
Key string `json:"key"`
|
|
Count int64 `json:"count"`
|
|
Name string `json:"name,omitempty"`
|
|
}
|
|
|
|
type statTraffic struct {
|
|
Name string `json:"name"`
|
|
Received int64 `json:"received"`
|
|
Sent int64 `json:"sent"`
|
|
}
|
|
|
|
type statSummary struct {
|
|
Since string `json:"since"`
|
|
From time.Time `json:"from"`
|
|
Zones []statTraffic `json:"zones"`
|
|
Interfaces []statTraffic `json:"interfaces"`
|
|
Drops int64 `json:"drops"`
|
|
DropsByChain []statEntry `json:"drops_by_chain"`
|
|
Rules []statEntry `json:"rules"`
|
|
BuiltIns []statEntry `json:"builtins"`
|
|
PeakConnections int64 `json:"peak_connections"`
|
|
ConntrackMax int64 `json:"conntrack_max"`
|
|
BlockedSources []statEntry `json:"blocked_sources"`
|
|
BlockedPorts []statEntry `json:"blocked_ports"`
|
|
DevicesUp []statEntry `json:"devices_up"`
|
|
DevicesDown []statEntry `json:"devices_down"`
|
|
}
|
|
|
|
type statTimeline struct {
|
|
Step string `json:"step"`
|
|
Times []int64 `json:"times"`
|
|
Series []struct {
|
|
Key string `json:"key"`
|
|
Values []int64 `json:"values"`
|
|
} `json:"series"`
|
|
}
|
|
|
|
type statStatus struct {
|
|
Enabled bool `json:"enabled"`
|
|
Database string `json:"database"`
|
|
Size int64 `json:"size"`
|
|
LastSample *time.Time `json:"last_sample"`
|
|
SampleError string `json:"sample_error"`
|
|
FirstData *time.Time `json:"first_data"`
|
|
PerDevice bool `json:"per_device"`
|
|
Retention struct {
|
|
Minute, Hour, Day string
|
|
} `json:"retention"`
|
|
Latest *struct {
|
|
Conntrack int64 `json:"conntrack"`
|
|
ConntrackMax int64 `json:"conntrack_max"`
|
|
RSS int64 `json:"rss"`
|
|
CPUPermille int64 `json:"cpu_permille"`
|
|
MemUsed int64 `json:"mem_used"`
|
|
Load float64 `json:"load"`
|
|
} `json:"latest"`
|
|
}
|
|
|
|
func statsCmd() *cobra.Command {
|
|
var since string
|
|
var limit int
|
|
cmd := &cobra.Command{
|
|
Use: "stats", Short: "Traffic, drops, blocked sources and devices over time",
|
|
Long: `Sums up what the firewall did over a while (--since: 90m, 24h, 7d, 4w):
|
|
traffic per zone, packets dropped, the busiest rules, the most connections
|
|
at once, the sources blocked most and what they tried, and (with traffic
|
|
per device on) the devices that used the most.
|
|
|
|
hc stats --since 7d
|
|
hc stats traffic --since 24h
|
|
hc stats top devices --since 30d
|
|
hc stats rules --unused`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
s, err := get[statSummary]("/stats/summary", q("since", since, "limit", strconv.Itoa(limit)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return output(s, func(w io.Writer) { printSummary(w, s) })
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d, 4w…")
|
|
cmd.Flags().IntVar(&limit, "limit", 5, "entries per top list")
|
|
cmd.AddCommand(statsStatusCmd(), statsTrafficCmd(), statsTopCmd(), statsRulesCmd(), statsConnectionsCmd())
|
|
return cmd
|
|
}
|
|
|
|
func printSummary(w io.Writer, s statSummary) {
|
|
fmt.Fprintf(w, "Since %s (%s)\n\n", s.From.Local().Format("2006-01-02 15:04"), s.Since)
|
|
if len(s.Zones) > 0 {
|
|
fmt.Fprintln(w, "ZONE\tRECEIVED\tSENT")
|
|
for _, t := range s.Zones {
|
|
fmt.Fprintf(w, "%s\t%s\t%s\n", t.Name, humanBytes(t.Received), humanBytes(t.Sent))
|
|
}
|
|
fmt.Fprintln(w)
|
|
}
|
|
var chains []string
|
|
for _, e := range s.DropsByChain {
|
|
chains = append(chains, fmt.Sprintf("%s %s", e.Key, humanCount(uint64(e.Count))))
|
|
}
|
|
drops := humanCount(uint64(s.Drops))
|
|
if len(chains) > 0 {
|
|
drops += " (" + strings.Join(chains, ", ") + ")"
|
|
}
|
|
fmt.Fprintf(w, "Dropped:\t%s packets\n", drops)
|
|
if s.ConntrackMax > 0 {
|
|
fmt.Fprintf(w, "Connections:\tat most %d at once (of %d)\n", s.PeakConnections, s.ConntrackMax)
|
|
}
|
|
list := func(title string, es []statEntry, unit func(int64) string) {
|
|
if len(es) == 0 {
|
|
return
|
|
}
|
|
fmt.Fprintf(w, "\n%s\n", title)
|
|
for _, e := range es {
|
|
name := e.Key
|
|
if e.Name != "" {
|
|
name = e.Name
|
|
}
|
|
fmt.Fprintf(w, " %s\t%s\n", name, unit(e.Count))
|
|
}
|
|
}
|
|
packets := func(n int64) string { return humanCount(uint64(n)) + " packets" }
|
|
list("Busiest rules:", s.Rules, packets)
|
|
list("Built-ins and policies:", s.BuiltIns, packets)
|
|
list("Blocked most (logged drops):", s.BlockedSources, packets)
|
|
list("What they tried:", s.BlockedPorts, packets)
|
|
list("Devices, sent:", s.DevicesUp, humanBytes)
|
|
list("Devices, received:", s.DevicesDown, humanBytes)
|
|
}
|
|
|
|
func statsStatusCmd() *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "status", Short: "Whether statistics are collected, since when, and the database", Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
s, err := get[statStatus]("/stats/status", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return output(s, func(w io.Writer) {
|
|
if !s.Enabled {
|
|
fmt.Fprintln(w, "Statistics are off (stats.enabled in housecarl.yaml).")
|
|
return
|
|
}
|
|
fmt.Fprintf(w, "Collecting:\tyes\n")
|
|
if s.Database != "" {
|
|
fmt.Fprintf(w, "Database:\t%s (%s)\n", s.Database, humanBytes(s.Size))
|
|
} else {
|
|
fmt.Fprintf(w, "Database size:\t%s\n", humanBytes(s.Size))
|
|
}
|
|
if s.FirstData != nil {
|
|
fmt.Fprintf(w, "Since:\t%s\n", s.FirstData.Local().Format("2006-01-02 15:04"))
|
|
}
|
|
if s.LastSample != nil {
|
|
fmt.Fprintf(w, "Last sample:\t%s\n", s.LastSample.Local().Format("15:04:05"))
|
|
}
|
|
if s.SampleError != "" {
|
|
fmt.Fprintf(w, "Error:\t%s\n", s.SampleError)
|
|
}
|
|
fmt.Fprintf(w, "Kept:\tper minute %s, hourly %s, daily %s\n", s.Retention.Minute, s.Retention.Hour, s.Retention.Day)
|
|
fmt.Fprintf(w, "Per device:\t%s\n", map[bool]string{true: "on", false: "off (hc settings set --stats-per-device on)"}[s.PerDevice])
|
|
if l := s.Latest; l != nil {
|
|
if l.ConntrackMax > 0 {
|
|
fmt.Fprintf(w, "Connections now:\t%d of %d\n", l.Conntrack, l.ConntrackMax)
|
|
}
|
|
fmt.Fprintf(w, "housecarld:\t%s, CPU %.1f%%\n", humanBytes(l.RSS), float64(l.CPUPermille)/10)
|
|
fmt.Fprintf(w, "Host:\t%s in use, load %.2f\n", humanBytes(l.MemUsed), l.Load)
|
|
}
|
|
})
|
|
},
|
|
}
|
|
}
|
|
|
|
func statsTrafficCmd() *cobra.Command {
|
|
var since, step string
|
|
var ifaces bool
|
|
cmd := &cobra.Command{
|
|
Use: "traffic", Short: "Bytes received from and sent to each zone (or interface) over time", Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
rx, tx := "zrx", "ztx"
|
|
if ifaces {
|
|
rx, tx = "rx", "tx"
|
|
}
|
|
tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", rx+":*,"+tx+":*"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return output(tl, func(w io.Writer) { printTimeline(w, tl, rx, tx) })
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
|
|
cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
|
|
cmd.Flags().BoolVar(&ifaces, "interfaces", false, "per interface instead of per zone")
|
|
return cmd
|
|
}
|
|
|
|
// printTimeline writes a row per bucket: each name's received and sent.
|
|
func printTimeline(w io.Writer, tl statTimeline, rx, tx string) {
|
|
var names []string
|
|
vals := map[string][]int64{}
|
|
for _, s := range tl.Series {
|
|
vals[s.Key] = s.Values
|
|
if n, ok := strings.CutPrefix(s.Key, rx+":"); ok {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
for _, s := range tl.Series {
|
|
if n, ok := strings.CutPrefix(s.Key, tx+":"); ok && vals[rx+":"+n] == nil {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
if len(names) == 0 {
|
|
fmt.Fprintln(w, "no traffic recorded yet")
|
|
return
|
|
}
|
|
fmt.Fprint(w, "TIME")
|
|
for _, n := range names {
|
|
fmt.Fprintf(w, "\t%s ↓\t%s ↑", n, n)
|
|
}
|
|
fmt.Fprintln(w)
|
|
layout := "01-02 15:04"
|
|
if tl.Step == "1d" {
|
|
layout = "2006-01-02"
|
|
}
|
|
at := func(k string, i int) int64 {
|
|
if v := vals[k]; i < len(v) {
|
|
return v[i]
|
|
}
|
|
return 0
|
|
}
|
|
for i, t := range tl.Times {
|
|
fmt.Fprint(w, time.Unix(t, 0).Local().Format(layout))
|
|
for _, n := range names {
|
|
fmt.Fprintf(w, "\t%s\t%s", humanBytes(at(rx+":"+n, i)), humanBytes(at(tx+":"+n, i)))
|
|
}
|
|
fmt.Fprintln(w)
|
|
}
|
|
}
|
|
|
|
func statsTopCmd() *cobra.Command {
|
|
var since string
|
|
var limit int
|
|
cmd := &cobra.Command{
|
|
Use: "top devices|blocked|tried|quota RULE-ID", Short: "The largest: devices by traffic, sources blocked, what they tried, a quota's users",
|
|
Long: `Top lists over the last while:
|
|
|
|
devices inside addresses by traffic to and from external zones (needs
|
|
hc settings set --stats-per-device on)
|
|
blocked sources of logged drops (the policies' drops with 'log drops'
|
|
on, and rules that log)
|
|
tried what those drops tried: tcp/22, udp/53, icmp/echo-request
|
|
quota ID who used a per-address quota of rule ID`,
|
|
Args: cobra.RangeArgs(1, 2),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
get1 := func(kind string) ([]statEntry, error) {
|
|
return get[[]statEntry]("/stats/top", q("kind", kind, "since", since, "limit", strconv.Itoa(limit)))
|
|
}
|
|
switch args[0] {
|
|
case "devices":
|
|
up, err := get1("device_up")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
down, err := get1("device_down")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out := map[string][2]int64{}
|
|
var order []string
|
|
for _, l := range [][]statEntry{down, up} {
|
|
for _, e := range l {
|
|
if _, ok := out[e.Key]; !ok {
|
|
out[e.Key] = [2]int64{}
|
|
order = append(order, e.Key)
|
|
}
|
|
}
|
|
}
|
|
for _, e := range down {
|
|
v := out[e.Key]
|
|
v[0] = e.Count
|
|
out[e.Key] = v
|
|
}
|
|
for _, e := range up {
|
|
v := out[e.Key]
|
|
v[1] = e.Count
|
|
out[e.Key] = v
|
|
}
|
|
names := lookupNames(order...)
|
|
return output(map[string]any{"up": up, "down": down}, func(w io.Writer) {
|
|
fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT")
|
|
for _, k := range order {
|
|
fmt.Fprintf(w, "%s\t%s\t%s\n", named(names, k, k), humanBytes(out[k][0]), humanBytes(out[k][1]))
|
|
}
|
|
})
|
|
case "blocked", "tried":
|
|
kind := map[string]string{"blocked": "blocked_src", "tried": "blocked_port"}[args[0]]
|
|
es, err := get1(kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var names map[string]string
|
|
if args[0] == "blocked" {
|
|
names = lookupNames(keys(es)...)
|
|
}
|
|
return output(es, func(w io.Writer) {
|
|
fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS")
|
|
for _, e := range es {
|
|
fmt.Fprintf(w, "%s\t%d\n", named(names, e.Key, e.Key), e.Count)
|
|
}
|
|
})
|
|
case "quota":
|
|
if len(args) != 2 {
|
|
return fmt.Errorf("give the rule's ID: hc stats top quota 12")
|
|
}
|
|
es, err := get1("quota_r" + args[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
names := lookupNames(keys(es)...)
|
|
return output(es, func(w io.Writer) {
|
|
fmt.Fprintln(w, "ADDRESS\tUSED")
|
|
for _, e := range es {
|
|
fmt.Fprintf(w, "%s\t%s\n", named(names, e.Key, e.Key), humanBytes(e.Count))
|
|
}
|
|
})
|
|
}
|
|
return fmt.Errorf("unknown list %q: devices, blocked, tried or quota", args[0])
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
|
|
cmd.Flags().IntVar(&limit, "limit", 20, "at most this many")
|
|
return cmd
|
|
}
|
|
|
|
// keys are a top list's keys.
|
|
func keys(es []statEntry) []string {
|
|
var out []string
|
|
for _, e := range es {
|
|
out = append(out, e.Key)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func statsRulesCmd() *cobra.Command {
|
|
var since string
|
|
var unused bool
|
|
cmd := &cobra.Command{
|
|
Use: "rules", Short: "Packets each rule, built-in and policy counted; --unused: rules that matched nothing", Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
es, err := get[[]statEntry]("/stats/hits", q("since", since))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if unused {
|
|
var keep []statEntry
|
|
for _, e := range es {
|
|
if e.Count == 0 && strings.HasPrefix(e.Key, "r") {
|
|
keep = append(keep, e)
|
|
}
|
|
}
|
|
es = keep
|
|
}
|
|
return output(es, func(w io.Writer) {
|
|
if unused && len(es) == 0 {
|
|
fmt.Fprintf(w, "every rule matched something in the last %s\n", since)
|
|
return
|
|
}
|
|
fmt.Fprintln(w, "WHAT\tPACKETS")
|
|
for _, e := range es {
|
|
fmt.Fprintf(w, "%s\t%d\n", orDash(e.Name), e.Count)
|
|
}
|
|
})
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&since, "since", "30d", "the last 24h, 7d, 30d…")
|
|
cmd.Flags().BoolVar(&unused, "unused", false, "only the rules that matched nothing")
|
|
return cmd
|
|
}
|
|
|
|
func statsConnectionsCmd() *cobra.Command {
|
|
var since, step string
|
|
cmd := &cobra.Command{
|
|
Use: "connections", Short: "How many connections the firewall tracked over time, against the limit", Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", "g:conntrack,g:conntrack_max"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return output(tl, func(w io.Writer) {
|
|
fmt.Fprintln(w, "TIME\tCONNECTIONS\tLIMIT")
|
|
for i, t := range tl.Times {
|
|
n, m := tl.Series[0].Values[i], tl.Series[1].Values[i]
|
|
if n < 0 {
|
|
continue // no sample
|
|
}
|
|
fmt.Fprintf(w, "%s\t%d\t%d\n", time.Unix(t, 0).Local().Format("01-02 15:04"), n, m)
|
|
}
|
|
})
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
|
|
cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
|
|
return cmd
|
|
}
|