diff --git a/CHANGELOG.md b/CHANGELOG.md index f3092a3..4c9983f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ All notable changes to Housecarl are documented here. The format follows ### Added +- **Names beside inside addresses.** Connections, the log, quota usage, + the dashboard's quotas and the statistics' devices and blocked + sources show each inside address's name, found by a reverse lookup: + `laptop 192.168.1.20` in the web UI, `192.168.1.20 (laptop)` in `hc`. + Only inside addresses are looked up (an inside zone's networks, and + private ones outside the external zones'). Names are cached for 10 + minutes and never stored. `names.enabled`, `names.resolver` and + `names.cache` in housecarl.yaml; `hc --no-names`; `POST /names`. - **Quotas on the dashboard.** With any rule holding a quota, the dashboard lists them all: the allowance, what is used (for a per-address quota, how many addresses and the busiest) with a bar, and @@ -62,6 +70,8 @@ All notable changes to Housecarl are documented here. The format follows ### Fixed +- `hc stats top devices` listed each device twice (once for what it + received, once for what it sent); it is one row with both again. - `hc rule add --before/--after/--first` made two changes (create at the end, then move), so two history entries, two probations, and the rule briefly in force at the end of its chain; staged in a changeset, the diff --git a/cmd/hc/firewall.go b/cmd/hc/firewall.go index 839940b..031ee83 100644 --- a/cmd/hc/firewall.go +++ b/cmd/hc/firewall.go @@ -298,6 +298,13 @@ func quotasCmd() *cobra.Command { if err != nil { return err } + var addrs []string + for _, q := range qs { + for _, a := range q.Addresses { + addrs = append(addrs, a.Address) + } + } + names := lookupNames(addrs...) return output(qs, func(w io.Writer) { fmt.Fprintln(w, "RULE\tUSED\tOF\tRESETS\tNEXT RESET") for _, q := range qs { @@ -317,7 +324,7 @@ func quotasCmd() *cobra.Command { } fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Rule, humanBytes(q.Used), of, resets, next) for _, a := range q.Addresses { - fmt.Fprintf(w, " %s\t%s\t\t\t\n", a.Address, humanBytes(a.Used)) + fmt.Fprintf(w, " %s\t%s\t\t\t\n", named(names, a.Address, a.Address), humanBytes(a.Used)) } } }) @@ -341,6 +348,11 @@ func logCmd() *cobra.Command { var last time.Time print := func(es []api.LogEntry) { sort.Slice(es, func(i, j int) bool { return es[i].Time.Before(es[j].Time) }) + var addrs []string + for _, e := range es { + addrs = append(addrs, e.Source, e.Dest) + } + names := lookupNames(addrs...) for _, e := range es { last = e.Time src, dst := e.Source, e.Dest @@ -350,6 +362,7 @@ func logCmd() *cobra.Command { if e.DestPort != 0 { dst = joinPort(dst, e.DestPort) } + src, dst = named(names, e.Source, src), named(names, e.Dest, dst) what := e.Protocol if e.ICMPType != "" { what += " " + e.ICMPType diff --git a/cmd/hc/main.go b/cmd/hc/main.go index 33061cd..23a741f 100644 --- a/cmd/hc/main.go +++ b/cmd/hc/main.go @@ -46,6 +46,7 @@ var ( flagCACert string flagQuiet bool flagNoConfirm bool + flagNoNames bool // lastClient is the client the command made (to confirm a change by // itself after it). lastClient *client.Client @@ -182,6 +183,7 @@ Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found, pf.StringVar(&flagCACert, "ca-cert", "", "PEM file with a CA to trust for the server's certificate") pf.BoolVarP(&flagQuiet, "quiet", "q", false, "print only errors (and IDs of new objects)") pf.BoolVar(&flagNoConfirm, "no-confirm", false, "leave a change on probation instead of confirming it once this client reaches Housecarl again") + pf.BoolVar(&flagNoNames, "no-names", false, "show inside addresses without their names (reverse lookups)") pf.Int64VarP(&flagChangeset, "changeset", "C", 0, "stage configuration changes in this changeset instead of applying them (env HOUSECARL_CHANGESET)") root.AddGroup(&cobra.Group{ID: "fw", Title: "Firewall:"}, &cobra.Group{ID: "changes", Title: "Changes:"}, &cobra.Group{ID: "admin", Title: "Administration:"}) add := func(group string, cmds ...*cobra.Command) { diff --git a/cmd/hc/objects.go b/cmd/hc/objects.go index 1bd97a4..346f60e 100644 --- a/cmd/hc/objects.go +++ b/cmd/hc/objects.go @@ -49,6 +49,46 @@ func counters() map[string]api.Counter { return cs.Counters } +// lookupNames asks for inside addresses' names, for tables: none with +// --no-names, another output format, or when they can't be had. +func lookupNames(addrs ...string) map[string]string { + out := map[string]string{} + if flagNoNames || flagOutput != "table" { + return out + } + seen := map[string]bool{} + var list []string + for _, a := range addrs { + if a != "" && !seen[a] { + seen[a] = true + list = append(list, a) + } + } + if len(list) == 0 { + return out + } + c, err := newClient() + if err != nil { + return out + } + var ns api.Names + if c.Do(ctx(), "POST", "/names", nil, api.NamesInput{Addresses: list}, &ns) != nil { + return out + } + for a, n := range ns.Names { + out[a] = strings.SplitN(n, ".", 2)[0] + } + return out +} + +// named adds an address's name to text (the address, or address:port). +func named(names map[string]string, addr, text string) string { + if n := names[addr]; n != "" { + return text + " (" + n + ")" + } + return text +} + func hits(c map[string]api.Counter, marks ...string) string { var p uint64 found := false diff --git a/cmd/hc/stats.go b/cmd/hc/stats.go index 4a13449..ceddad4 100644 --- a/cmd/hc/stats.go +++ b/cmd/hc/stats.go @@ -283,6 +283,7 @@ func statsTopCmd() *cobra.Command { for _, l := range [][]statEntry{down, up} { for _, e := range l { if _, ok := out[e.Key]; !ok { + out[e.Key] = [2]int64{} order = append(order, e.Key) } } @@ -297,10 +298,11 @@ func statsTopCmd() *cobra.Command { v[1] = e.Count out[e.Key] = v } + names := lookupNames(order...) return output(map[string]any{"up": up, "down": down}, func(w io.Writer) { fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT") for _, k := range order { - fmt.Fprintf(w, "%s\t%s\t%s\n", k, humanBytes(out[k][0]), humanBytes(out[k][1])) + fmt.Fprintf(w, "%s\t%s\t%s\n", named(names, k, k), humanBytes(out[k][0]), humanBytes(out[k][1])) } }) case "blocked", "tried": @@ -309,10 +311,14 @@ func statsTopCmd() *cobra.Command { if err != nil { return err } + var names map[string]string + if args[0] == "blocked" { + names = lookupNames(keys(es)...) + } return output(es, func(w io.Writer) { fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS") for _, e := range es { - fmt.Fprintf(w, "%s\t%d\n", e.Key, e.Count) + fmt.Fprintf(w, "%s\t%d\n", named(names, e.Key, e.Key), e.Count) } }) case "quota": @@ -323,10 +329,11 @@ func statsTopCmd() *cobra.Command { if err != nil { return err } + names := lookupNames(keys(es)...) return output(es, func(w io.Writer) { fmt.Fprintln(w, "ADDRESS\tUSED") for _, e := range es { - fmt.Fprintf(w, "%s\t%s\n", e.Key, humanBytes(e.Count)) + fmt.Fprintf(w, "%s\t%s\n", named(names, e.Key, e.Key), humanBytes(e.Count)) } }) } @@ -338,6 +345,15 @@ func statsTopCmd() *cobra.Command { return cmd } +// keys are a top list's keys. +func keys(es []statEntry) []string { + var out []string + for _, e := range es { + out = append(out, e.Key) + } + return out +} + func statsRulesCmd() *cobra.Command { var since string var unused bool diff --git a/cmd/hc/traffic.go b/cmd/hc/traffic.go index 97f9ddc..d77a74e 100644 --- a/cmd/hc/traffic.go +++ b/cmd/hc/traffic.go @@ -38,10 +38,16 @@ rules as it stands. if err != nil { return err } + var addrs []string + for _, c := range cs { + addrs = append(addrs, c.Source, c.Destination) + } + names := lookupNames(addrs...) return output(cs, func(w io.Writer) { fmt.Fprintln(w, "PROTO\tFROM\tTO\tNAT\tSTATE\tEXPIRES") for _, c := range cs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", c.Protocol, hostPort(c.Source, c.SourcePort), hostPort(c.Destination, c.DestinationPort), natText(c), c.State, fmtSeconds(c.Timeout)) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", c.Protocol, named(names, c.Source, hostPort(c.Source, c.SourcePort)), + named(names, c.Destination, hostPort(c.Destination, c.DestinationPort)), natText(c), c.State, fmtSeconds(c.Timeout)) } }) }, diff --git a/cmd/housecarld/main.go b/cmd/housecarld/main.go index c022327..139503a 100644 --- a/cmd/housecarld/main.go +++ b/cmd/housecarld/main.go @@ -32,6 +32,7 @@ import ( "housecarl/internal/jarl" "housecarl/internal/linkwatch" "housecarl/internal/model" + "housecarl/internal/names" "housecarl/internal/nflog" "housecarl/internal/nft" "housecarl/internal/render" @@ -176,11 +177,15 @@ func serveCmd() *cobra.Command { acme = certs.New(ctx, certs.Config{ACME: a.cfg.TLS.ACME, Version: version.Short()}, jc, a.store, a.log) go acme.Run(ctx) } + var resolver *names.Resolver + if a.cfg.Names.Enabled { + resolver = names.New(a.cfg.Names.Resolver, time.Duration(a.cfg.Names.Cache)) + } srv := server.New(server.Config{ Listen: a.cfg.Listen, TLSCertFile: a.cfg.TLS.CertFile, TLSKeyFile: a.cfg.TLS.KeyFile, TrustedProxies: a.cfg.Proxies(), SessionLifetime: time.Duration(a.cfg.Session.Lifetime), SessionIdle: time.Duration(a.cfg.Session.IdleTimeout), CookieSecure: a.cfg.Session.CookieSecure, Socket: a.cfg.Socket, ACME: acme, - Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position, + Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position, Names: resolver, }, eng, fw, host, a.log, web.FS()) // Put the ruleset in force: what was on probation when // housecarld stopped is rolled back first. An empty diff --git a/docs/api.md b/docs/api.md index 9e2490a..dcbb8ca 100644 --- a/docs/api.md +++ b/docs/api.md @@ -82,6 +82,7 @@ Creating answers 201, other changes 200. | `GET /counters` | `{"since", "counters": {"r12": {"packets", "bytes"}, ...}}`, by mark ([design.md](design.md#its-own-table-only)); totals since housecarld started | | `GET /quotas` | each rule's quota: `rule_id`, `rule`, `quota`, `used` (bytes), `addresses` (`[{"address", "used"}]` for a per-address quota), `last_reset`, `next_reset` | | `GET /connections[?q=&limit=]` | tracked connections: `family`, `protocol`, `source`, `source_port`, `destination`, `destination_port` (or `icmp_id`, `icmp_type`, `icmp_code`), `dnat_to`, `dnat_port`, `snat_to`, `state`, `assured`, `timeout` (seconds left), `packets` and `bytes` when the kernel counts them. `q`: all of its words in the addresses, `:port`, protocol, state | +| `POST /names` | `{"addresses": [...]}` (at most 2000): `{"enabled", "names": {"192.168.1.20": "laptop.example.net"}}`, the names of the inside addresses among them, by reverse lookup ([configuration.md](configuration.md#names)). Addresses without a name, outside ones, and those still being looked up after a second are left out; ask again for those. `log:read` | | `POST /connections/close` | `{"connections": [...]}` (as listed) or `{"search": "192.168.1.50"}`: delete their tracking entries (`config:write`, audited); answers `{"closed": N}`. 409 in shadow mode | | `POST /trace` | `{"address", "port", "protocol", "seconds"}` (5-600, default 60): start tracing (`config:write`, audited); 409 while one runs | | `GET /trace` | the trace running, or the last: `running`, `input`, `started`, `ends`, `error`, `packets`: each with `time`, `in`, `out`, `protocol`, addresses and ports, `tcp_flags`, `icmp_type`, `steps` (`table`, `chain`, `kind` rule or policy, `rule`, `mark`, `label`, `verdict`), `verdict` (accepted, dropped) and `decided_by`; in shadow mode `would_drop`, the step where Housecarl would have dropped it | diff --git a/docs/cli.md b/docs/cli.md index 9c4850c..ed24790 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -32,6 +32,7 @@ command and its flags. | `-C, --changeset ID` | `HOUSECARL_CHANGESET` (checked, see [Changesets](#changesets)) | stage configuration changes in this changeset instead of making them | | `-o, --output FORMAT` | | `table` (default), `json`, `yaml` or `id` | | `-q, --quiet` | | print only errors (and the IDs of new objects) | +| `--no-names` | | show inside addresses without their names: tables show `192.168.1.20 (laptop)` ([configuration.md](configuration.md#names)) | Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found, 5 conflict, 6 invalid or refused by validation or nftables, 7 diff --git a/docs/configuration.md b/docs/configuration.md index 142b58d..b75dfec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,12 +32,34 @@ reads the file when it starts. | `stats.top_n` | `100` | entries kept per top list, per hour and per day | | `shadow.enabled` | `false` | shadow mode: Housecarl's ruleset is loaded beside the firewall you have, but blocks, rejects and translates nothing; its counters, log and tracer show what it would do. No change is on probation, and takeovers and closing connections are refused ([shadow.md](shadow.md)) | | `shadow.position` | `after` | where Housecarl's filter chains run in shadow mode: `after` the other firewall's (`filter + 10`: it sees only what that one lets through, so what it would drop is what switching would break) or `before` them (`filter - 10`: it sees every packet) | +| `names.enabled` | `true` | show inside addresses' names beside them, found by reverse lookups (below) | +| `names.resolver` | `""` | the DNS server to ask, such as `10.10.0.1:53`; empty: the system's (`/etc/hosts`, then `/etc/resolv.conf`) | +| `names.cache` | `10m` | how long a name, or the lack of one, is kept | Durations are written `15s`, `12h` or `30d`. Shadow mode is here, and not a firewall setting, so that no user or API token can switch enforcement off; restart housecarld to switch. +### Names + +The web UI and `hc` show a name beside each **inside** address in +connections, the log, quotas and traffic per device: `laptop +192.168.1.20`. Housecarl finds it with a reverse (PTR) lookup, so the +names are your reverse zones' (and `/etc/hosts`'). An address counts as +inside when it is in a network connected to an inside zone's interface, +or private (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`) +and not in an external zone's network. Outside addresses get no name: +their reverse zones belong to whoever holds the addresses, who would +choose the names, and looking up every scanner is slow. + +Names are looked up when a page or `hc` asks, cached (a failed lookup is +tried again after a minute), and never stored: statistics stay by +address. With a split DNS (views), the server asked must answer the +firewall as an inside client; set `names.resolver` if the system's +resolver doesn't. IPv6 traffic per device is counted per /64, which has +no name. `hc --no-names` leaves them out. + The probation's length is a firewall setting, not here: `hc settings set --probation 5m`. So is counting traffic per device for the statistics: `hc settings set --stats-per-device on`. diff --git a/internal/api/types.go b/internal/api/types.go index 8caefb5..2238b26 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -384,6 +384,19 @@ type Connection struct { Bytes uint64 `json:"bytes,omitempty"` } +// NamesInput asks for the names of addresses (POST /names). +type NamesInput struct { + Addresses []string `json:"addresses"` +} + +// Names are the names found for the inside addresses asked about, by +// address; those without one (or still being looked up) are left out. +// Enabled is false when names are off (names.enabled). +type Names struct { + Enabled bool `json:"enabled"` + Names map[string]string `json:"names"` +} + // CloseConnections closes connections: these, or every one Search finds // (as GET /connections?q= does). type CloseConnections struct { diff --git a/internal/config/config.go b/internal/config/config.go index 9e5927f..dde117f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,6 +44,19 @@ type Config struct { // Shadow loads the ruleset without letting it block, reject or // translate anything, beside the firewall in force (docs/shadow.md). Shadow Shadow `yaml:"shadow"` + // Names shows inside addresses' names, by reverse lookups + // (docs/configuration.md). + Names Names `yaml:"names"` +} + +// Names configures the reverse lookups that put names beside inside +// addresses in the web UI and hc. +type Names struct { + Enabled bool `yaml:"enabled"` + // Resolver is a DNS server ("10.10.0.1:53"; "": the system's). + Resolver string `yaml:"resolver"` + // Cache is how long a name is kept. + Cache Duration `yaml:"cache"` } // Shadow is shadow mode: Housecarl's ruleset is loaded and counts, logs @@ -224,6 +237,7 @@ func Default() *Config { Stats: Stats{Enabled: true, Database: "/var/lib/housecarl/stats.db", TopN: 100, Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)}}, Shadow: Shadow{Position: ShadowAfter}, + Names: Names{Enabled: true, Cache: Duration(10 * time.Minute)}, TLS: TLS{ACME: ACME{Directory: "letsencrypt-staging", Resolvers: []string{"1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"}, CheckPropagation: true}}, } } @@ -298,6 +312,14 @@ func (c *Config) check() error { if c.Shadow.Position != ShadowAfter && c.Shadow.Position != ShadowBefore { return fmt.Errorf("shadow.position must be after or before") } + if c.Names.Resolver != "" { + if _, err := netip.ParseAddrPort(c.Names.Resolver); err != nil { + return fmt.Errorf("names.resolver must be an address and port, such as 10.10.0.1:53") + } + } + if c.Names.Cache <= 0 { + return fmt.Errorf("names.cache must be a duration, such as 10m") + } return nil } diff --git a/internal/names/names.go b/internal/names/names.go new file mode 100644 index 0000000..15b933f --- /dev/null +++ b/internal/names/names.go @@ -0,0 +1,152 @@ +// Package names finds the names of addresses by reverse lookups, for +// showing them beside the addresses (connections, the log, quotas, +// traffic per device). Names are cached and never stored: statistics stay +// keyed by address. +package names + +import ( + "context" + "errors" + "net" + "net/netip" + "strings" + "sync" + "time" +) + +// Lookups running at once, and the most names kept. +const ( + parallel = 16 + maxCache = 20000 +) + +// Resolver looks up and caches addresses' names. +type Resolver struct { + lookup func(ctx context.Context, addr string) ([]string, error) + // ttl is how long a name (or its absence) is kept; a lookup that + // failed (a timeout, a server error) is tried again after retry. + ttl, retry time.Duration + timeout time.Duration + now func() time.Time + + mu sync.Mutex + cache map[netip.Addr]entry + pending map[netip.Addr]chan struct{} + sem chan struct{} +} + +type entry struct { + name string + expires time.Time +} + +// New makes a resolver asking server ("10.10.0.1:53"; "" for the +// system's, /etc/hosts then /etc/resolv.conf), keeping names for ttl. +func New(server string, ttl time.Duration) *Resolver { + res := &net.Resolver{PreferGo: true} + if server != "" { + res.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, server) + } + } + return NewFunc(res.LookupAddr, ttl) +} + +// NewFunc makes a resolver with its own lookup (tests). +func NewFunc(lookup func(ctx context.Context, addr string) ([]string, error), ttl time.Duration) *Resolver { + return &Resolver{lookup: lookup, ttl: ttl, retry: time.Minute, timeout: 2 * time.Second, now: time.Now, + cache: map[netip.Addr]entry{}, pending: map[netip.Addr]chan struct{}{}, sem: make(chan struct{}, parallel)} +} + +// Names returns the names of addrs: those cached, and those looked up +// within wait. Lookups still running then carry on and are cached for +// the next call. An address without a name is left out. +func (r *Resolver) Names(ctx context.Context, addrs []netip.Addr, wait time.Duration) map[netip.Addr]string { + out := map[netip.Addr]string{} + var waiting []netip.Addr + var done []chan struct{} + now := r.now() + r.mu.Lock() + for _, a := range addrs { + if e, ok := r.cache[a]; ok && now.Before(e.expires) { + if e.name != "" { + out[a] = e.name + } + continue + } + ch, ok := r.pending[a] + if !ok { + ch = make(chan struct{}) + r.pending[a] = ch + go r.resolve(a, ch) + } + waiting = append(waiting, a) + done = append(done, ch) + } + r.mu.Unlock() + if len(waiting) == 0 { + return out + } + t := time.NewTimer(wait) + defer t.Stop() + for _, ch := range done { + select { + case <-ch: + case <-t.C: + return r.collect(out, waiting) + case <-ctx.Done(): + return r.collect(out, waiting) + } + } + return r.collect(out, waiting) +} + +// collect adds what the lookups found for addrs. +func (r *Resolver) collect(out map[netip.Addr]string, addrs []netip.Addr) map[netip.Addr]string { + r.mu.Lock() + defer r.mu.Unlock() + for _, a := range addrs { + if e, ok := r.cache[a]; ok && e.name != "" { + out[a] = e.name + } + } + return out +} + +func (r *Resolver) resolve(a netip.Addr, done chan struct{}) { + r.sem <- struct{}{} + ctx, cancel := context.WithTimeout(context.Background(), r.timeout) + ns, err := r.lookup(ctx, a.String()) + cancel() + <-r.sem + e := entry{expires: r.now().Add(r.ttl)} + var dnsErr *net.DNSError + switch { + case err == nil && len(ns) > 0: + e.name = strings.TrimSuffix(ns[0], ".") + case err != nil && !(errors.As(err, &dnsErr) && dnsErr.IsNotFound): + e.expires = r.now().Add(r.retry) + } + r.mu.Lock() + if len(r.cache) >= maxCache { + r.prune() + } + r.cache[a] = e + delete(r.pending, a) + r.mu.Unlock() + close(done) +} + +// prune drops expired names, or all of them when none has expired. +func (r *Resolver) prune() { + now := r.now() + for a, e := range r.cache { + if !now.Before(e.expires) { + delete(r.cache, a) + } + } + if len(r.cache) >= maxCache { + clear(r.cache) + } +} diff --git a/internal/names/names_test.go b/internal/names/names_test.go new file mode 100644 index 0000000..2434248 --- /dev/null +++ b/internal/names/names_test.go @@ -0,0 +1,57 @@ +package names + +import ( + "context" + "errors" + "net" + "net/netip" + "sync/atomic" + "testing" + "time" +) + +func TestNames(t *testing.T) { + var calls atomic.Int32 + slow := make(chan struct{}) + r := NewFunc(func(ctx context.Context, addr string) ([]string, error) { + calls.Add(1) + switch addr { + case "10.0.0.1": + return []string{"nas.example.net."}, nil + case "10.0.0.2": + return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true} + case "10.0.0.3": + return nil, errors.New("server failure") + case "10.0.0.4": + <-slow + return []string{"slow.example.net."}, nil + } + return nil, nil + }, time.Hour) + a := netip.MustParseAddr + all := []netip.Addr{a("10.0.0.1"), a("10.0.0.2"), a("10.0.0.3"), a("10.0.0.4")} + + got := r.Names(context.Background(), all, 200*time.Millisecond) + if len(got) != 1 || got[a("10.0.0.1")] != "nas.example.net" { + t.Fatalf("first call: %v", got) + } + // The slow lookup finishes later and is there next time; the others + // come from the cache, the failure too until it is retried. + close(slow) + time.Sleep(50 * time.Millisecond) + got = r.Names(context.Background(), all, 200*time.Millisecond) + if len(got) != 2 || got[a("10.0.0.4")] != "slow.example.net" { + t.Fatalf("second call: %v", got) + } + if n := calls.Load(); n != 4 { + t.Fatalf("%d lookups, want 4 (cached)", n) + } + + // A failure is tried again after retry, a missing name after ttl. + now := time.Now().Add(2 * time.Minute) + r.now = func() time.Time { return now } + r.Names(context.Background(), all, 200*time.Millisecond) + if n := calls.Load(); n != 5 { + t.Fatalf("%d lookups, want 5 (the failure again)", n) + } +} diff --git a/internal/server/h_traffic.go b/internal/server/h_traffic.go index 23bb24d..d9688a5 100644 --- a/internal/server/h_traffic.go +++ b/internal/server/h_traffic.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/netip" "sort" "strconv" "strings" @@ -88,6 +89,83 @@ func (s *Server) listConnections(w http.ResponseWriter, r *http.Request, p *auth return nil } +// lookupNames finds the names of the inside addresses asked about +// (docs/configuration.md#names), waiting at most a second for lookups. +func (s *Server) lookupNames(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := require(p, auth.ResLog, false); err != nil { + return err + } + var in api.NamesInput + if err := decodeJSON(r, &in); err != nil { + return err + } + if len(in.Addresses) > maxNames { + return badRequest("ask for at most %d addresses at once", maxNames) + } + out := api.Names{Enabled: s.cfg.Names != nil, Names: map[string]string{}} + if s.cfg.Names == nil { + writeJSON(w, http.StatusOK, out) + return nil + } + inside := s.insideNetworks(s.eng.State()) + asked := map[netip.Addr][]string{} + var addrs []netip.Addr + for _, a := range in.Addresses { + ad, err := netip.ParseAddr(a) + if err != nil || !inside(ad.Unmap()) { + continue + } + ad = ad.Unmap() + if _, ok := asked[ad]; !ok { + addrs = append(addrs, ad) + } + asked[ad] = append(asked[ad], a) + } + for a, n := range s.cfg.Names.Names(r.Context(), addrs, time.Second) { + for _, as := range asked[a] { + out.Names[as] = n + } + } + writeJSON(w, http.StatusOK, out) + return nil +} + +// maxNames is the most addresses one POST /names may ask about. +const maxNames = 2000 + +// insideNetworks tells inside addresses: in a network connected to an +// inside zone's interface, or private and not in an external zone's. +func (s *Server) insideNetworks(st *model.State) func(netip.Addr) bool { + external := map[string]bool{} + for _, z := range st.Zones { + external[z.Name] = z.External + } + var in, out []netip.Prefix + for p, zone := range s.connected(st) { + if external[zone] { + out = append(out, p) + } else { + in = append(in, p) + } + } + return func(a netip.Addr) bool { + if a.IsLoopback() || a.IsUnspecified() || a.IsMulticast() || a.IsLinkLocalUnicast() { + return false + } + for _, p := range in { + if p.Contains(a) { + return true + } + } + for _, p := range out { + if p.Contains(a) { + return false + } + } + return a.IsPrivate() + } +} + // closeConnections deletes tracking entries: the rules check those // connections again with their next packet. func (s *Server) closeConnections(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { diff --git a/internal/server/routes.go b/internal/server/routes.go index ce3ccb3..fac52a5 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -41,6 +41,7 @@ func (s *Server) routes() { m.HandleFunc("GET "+p+"/quotas", s.authed(s.listQuotas)) m.HandleFunc("GET "+p+"/connections", s.authed(s.listConnections)) m.HandleFunc("POST "+p+"/connections/close", s.authed(s.closeConnections)) + m.HandleFunc("POST "+p+"/names", s.authed(s.lookupNames)) m.HandleFunc("GET "+p+"/trace", s.authed(s.getTrace)) m.HandleFunc("POST "+p+"/trace", s.authed(s.startTrace)) m.HandleFunc("DELETE "+p+"/trace", s.authed(s.stopTrace)) diff --git a/internal/server/server.go b/internal/server/server.go index 26280f1..0da4c84 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -23,6 +23,7 @@ import ( "housecarl/internal/certs" "housecarl/internal/engine" "housecarl/internal/model" + "housecarl/internal/names" "housecarl/internal/nft" "housecarl/internal/render" "housecarl/internal/stats" @@ -51,6 +52,8 @@ type Config struct { // after or before, for the status. Shadow bool ShadowPosition string + // Names looks up inside addresses' names (nil: names are off). + Names *names.Resolver } // Interfaces lists the host's interfaces with their addresses, and diff --git a/internal/server/server_test.go b/internal/server/server_test.go index eda5033..3102592 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "net/netip" "os" + "reflect" "strings" "sync" "testing" @@ -21,6 +22,7 @@ import ( "housecarl/internal/auth" "housecarl/internal/engine" "housecarl/internal/model" + "housecarl/internal/names" "housecarl/internal/nft" "housecarl/internal/render" "housecarl/internal/stats" @@ -982,6 +984,41 @@ func TestConnections(t *testing.T) { } } +// TestNames: names only for inside addresses (an inside zone's network, +// or private outside the external zones'), under the address as asked. +func TestNames(t *testing.T) { + var asked []string + var mu sync.Mutex + e := newEnvWith(t, Config{Names: names.NewFunc(func(_ context.Context, a string) ([]string, error) { + mu.Lock() + asked = append(asked, a) + mu.Unlock() + return []string{"host-" + strings.ReplaceAll(a, ".", "-") + ".example.net."}, nil + }, time.Hour)}) + e.call(e.token, "POST", "/zones", model.Zone{Name: "wan", Interfaces: []string{"wan"}, External: true}, nil) + e.call(e.token, "POST", "/zones", model.Zone{Name: "lan", Interfaces: []string{"eth1"}}, nil) + var out api.Names + e.call(e.token, "POST", "/names", api.NamesInput{Addresses: []string{"10.0.10.5", "::ffff:10.0.10.5", "192.168.7.7", "217.155.8.194", "8.8.8.8", "fe80::1", "nonsense"}}, &out) + want := map[string]string{"10.0.10.5": "host-10-0-10-5.example.net", "::ffff:10.0.10.5": "host-10-0-10-5.example.net", "192.168.7.7": "host-192-168-7-7.example.net"} + if !out.Enabled || !reflect.DeepEqual(out.Names, want) { + t.Errorf("names: %+v", out) + } + if len(asked) != 2 { + t.Errorf("looked up %v, want the two inside addresses once each", asked) + } + var tok api.Token + e.call(e.token, "POST", "/me/tokens", api.TokenInput{Name: "monitor", Role: "log"}, &tok) + var one, none api.Names + if code := e.call(tok.Secret, "POST", "/names", api.NamesInput{Addresses: []string{"10.0.10.5"}}, &one); code != 200 || len(one.Names) != 1 { + t.Errorf("with log:read: %d %+v", code, one) + } + off := newEnv(t) + off.call(off.token, "POST", "/names", api.NamesInput{Addresses: []string{"10.0.10.5"}}, &none) + if none.Enabled || len(none.Names) != 0 { + t.Errorf("names off: %+v", none) + } +} + // TestTrace: a trace runs with the filter's table, collects packets named // after Housecarl's objects, and stops when asked; one at a time. func TestTrace(t *testing.T) { diff --git a/internal/web/static/assets/pages/connections.js b/internal/web/static/assets/pages/connections.js index 01c1998..b50e135 100644 --- a/internal/web/static/assets/pages/connections.js +++ b/internal/web/static/assets/pages/connections.js @@ -3,7 +3,7 @@ import { h, mount, input, debounce, toast, confirmDialog, badge } from "../ui.js"; import { get, api, can } from "../api.js"; import { onLeave } from "../app.js"; -import { fmtSeconds } from "../widgets.js"; +import { fmtSeconds, lookupNames, named } from "../widgets.js"; export async function render(el, params, query) { const search = input({ type: "search", placeholder: "address, :port, protocol, state…", value: query.get("q") || "" }); @@ -22,14 +22,15 @@ export async function render(el, params, query) { }; const draw = async () => { shown = await get("/connections", { q: search.value.trim(), limit: 500 }); + const names = await lookupNames(shown.flatMap((c) => [c.source, c.destination, c.dnat_to])); count.textContent = shown.length === 1 ? "1 connection" : shown.length + " connections"; mount(out, shown.length ? h("div", { class: "table-wrap" }, h("table", null, h("thead", null, h("tr", null, ["Protocol", "From", "To", "NAT", "State", "Expires", ""].map((t) => h("th", null, t)))), h("tbody", null, shown.map((c) => h("tr", null, h("td", null, c.protocol), - h("td", { class: "mono" }, hostPort(c.source, c.source_port)), - h("td", { class: "mono" }, hostPort(c.destination, c.destination_port)), - h("td", { class: "mono small" }, c.dnat_to ? "to " + hostPort(c.dnat_to, c.dnat_port || c.destination_port) : "", c.snat_to ? (c.dnat_to ? ", " : "") + "as " + c.snat_to : ""), + h("td", { class: "mono" }, named(names, c.source, hostPort(c.source, c.source_port))), + h("td", { class: "mono" }, named(names, c.destination, hostPort(c.destination, c.destination_port))), + h("td", { class: "mono small" }, c.dnat_to ? ["to ", named(names, c.dnat_to, hostPort(c.dnat_to, c.dnat_port || c.destination_port))] : "", c.snat_to ? (c.dnat_to ? ", " : "") + "as " + c.snat_to : ""), h("td", null, c.state || "", c.assured ? null : [" ", badge("one way", "")]), h("td", { class: "small" }, fmtSeconds(c.timeout)), h("td", { class: "actions-cell" }, write ? h("button", { class: "small reveal", onclick: () => close([c], "this connection") }, "Close") : null)))))) : diff --git a/internal/web/static/assets/pages/dashboard.js b/internal/web/static/assets/pages/dashboard.js index df2b7c1..3e0cff2 100644 --- a/internal/web/static/assets/pages/dashboard.js +++ b/internal/web/static/assets/pages/dashboard.js @@ -3,7 +3,7 @@ import { h, mount, table, badge, problemsList, time, plural } from "../ui.js"; import { get, can } from "../api.js"; import { onLeave } from "../app.js"; -import { describe, hits, humanCount, fmtBytes, usageBar } from "../widgets.js"; +import { describe, hits, humanCount, fmtBytes, usageBar, lookupNames, named } from "../widgets.js"; import { lineChart, chartCard, legend, timeTable, stepSeconds } from "../charts.js"; import { bytes } from "./stats.js"; @@ -55,7 +55,7 @@ export async function render(el) { { label: "Hits", render: (r) => hits(cs, "r" + r.id) }, { label: "Comment", render: (r) => r.comment }, ], busiest)) : null, - quotas.length ? quotaPanel(quotas) : null, + quotas.length ? quotaPanel(quotas, await lookupNames(quotas.map((q) => (q.addresses || [])[0]).filter(Boolean).map((a) => a.address))) : null, log.length ? h("div", { class: "panel" }, h("div", { class: "panel-head" }, h("h2", null, "Latest logged packets"), h("a", { class: "button small", href: "#/log" }, "The log")), table([ @@ -73,16 +73,16 @@ export async function render(el) { // quotaPanel lists every rule's quota with what it used: the total, or // for a per-address quota the busiest address (the rest are on the rule). -function quotaPanel(quotas) { +function quotaPanel(quotas, names) { const perAddress = (q) => q.quota.per_source || q.quota.per_destination; const used = (q) => perAddress(q) ? ((q.addresses || [])[0] || { used: 0 }).used : q.used; return h("div", { class: "panel" }, h("div", { class: "panel-head" }, h("h2", null, "Quotas"), h("a", { class: "button small", href: "#/stats" }, "Statistics")), table([ - { label: "Rule", render: (q) => h("a", { href: "#/rules?rule=" + q.rule_id }, "#" + q.rule_id + " ", q.rule) }, + { label: "Rule", render: (q) => h("a", { href: "#/rules?rule=" + q.rule_id }, q.rule) }, { label: "Allowance", render: (q) => fmtBytes(q.quota.bytes) + (q.quota.per_source ? " each source" : q.quota.per_destination ? " each destination" : "") }, { label: "Used", render: (q) => perAddress(q) - ? [plural((q.addresses || []).length, "address", "addresses"), q.addresses && q.addresses.length ? ", busiest " + fmtBytes(used(q)) : ""] + ? [plural((q.addresses || []).length, "address", "addresses"), q.addresses && q.addresses.length ? [", busiest ", named(names, q.addresses[0].address), " " + fmtBytes(used(q))] : ""] : fmtBytes(q.used) }, { label: "", render: (q) => usageBar(used(q), q.quota.bytes) }, { label: "Next reset", render: (q) => q.next_reset ? time(q.next_reset) : h("span", { class: "muted" }, "by hand") }, diff --git a/internal/web/static/assets/pages/log.js b/internal/web/static/assets/pages/log.js index 03ad5fe..b842ee4 100644 --- a/internal/web/static/assets/pages/log.js +++ b/internal/web/static/assets/pages/log.js @@ -3,6 +3,7 @@ import { h, mount, input, debounce, badge } from "../ui.js"; import { get, state } from "../api.js"; import { onLeave } from "../app.js"; +import { lookupNames, named } from "../widgets.js"; export async function render(el, params, query) { const search = input({ type: "search", placeholder: "address, interface, rule…", value: query.get("q") || "" }); @@ -12,6 +13,7 @@ export async function render(el, params, query) { const addr = (a, port) => port ? (a.includes(":") ? "[" + a + "]:" + port : a + ":" + port) : a; const draw = async () => { const es = await get("/log", { limit: 300, q: search.value.trim() }); + const names = await lookupNames(es.flatMap((e) => [e.source, e.destination])); mount(out, es.length ? h("div", { class: "table-wrap" }, h("table", { class: "activity" }, h("thead", null, h("tr", null, ["Time", "Why", "In", "Out", "From", "To", "Protocol"].map((t) => h("th", null, t)))), h("tbody", null, es.map((e) => h("tr", null, @@ -19,7 +21,8 @@ export async function render(el, params, query) { h("td", null, e.mark && e.mark.startsWith("r") ? h("a", { href: "#/rules?rule=" + e.mark.slice(1), title: "rule #" + e.mark.slice(1) }, e.prefix) : e.prefix, e.shadow && /^(drop|reject)$/.test(e.verdict) ? [" ", badge("would " + e.verdict, "warn")] : null), h("td", { class: "mono" }, e.in || ""), h("td", { class: "mono" }, e.out || ""), - h("td", { class: "mono" }, addr(e.source, e.source_port)), h("td", { class: "mono" }, addr(e.destination, e.destination_port)), + h("td", { class: "mono" }, named(names, e.source, addr(e.source, e.source_port))), + h("td", { class: "mono" }, named(names, e.destination, addr(e.destination, e.destination_port))), h("td", null, e.protocol, e.icmp_type ? " " + e.icmp_type : "", e.tcp_flags ? h("span", { class: "muted small" }, " " + e.tcp_flags) : null)))))) : h("div", { class: "empty" }, state.me.mode === "shadow" ? "Nothing logged yet. In shadow mode, what the policies would drop is logged; turn logging on for a rule with Edit → Log." diff --git a/internal/web/static/assets/pages/rules.js b/internal/web/static/assets/pages/rules.js index 912ad86..4682a88 100644 --- a/internal/web/static/assets/pages/rules.js +++ b/internal/web/static/assets/pages/rules.js @@ -2,7 +2,7 @@ import { h, mount, table, modal, errorBox, field, input, select, checkbox, confirmDialog, badge, textarea, toast } from "../ui.js"; import { get, api, change, can } from "../api.js"; import { rerender } from "../app.js"; -import { names, matchEditor, describe, hits, counters, listEditor, rowClass, live, fmtBytes, quotas, usageBar } from "../widgets.js"; +import { names, matchEditor, describe, hits, counters, listEditor, rowClass, live, fmtBytes, quotas, usageBar, lookupNames, named } from "../widgets.js"; const chains = [ ["input", "Input", "Traffic to the firewall itself."], @@ -179,10 +179,11 @@ function quotaLine(r, q) { } async function usageDialog(r, q) { + const names = await lookupNames(q.addresses.map((a) => a.address)); await modal("Quota usage of " + (r.comment || "rule #" + r.id), h("div", null, h("p", { class: "muted" }, "Each " + (r.quota.per_source ? "source" : "destination") + " has " + fmtBytes(r.quota.bytes) + "."), table([ - { label: "Address", render: (a) => h("span", { class: "mono" }, a.address) }, + { label: "Address", render: (a) => h("span", { class: "mono" }, named(names, a.address)) }, { label: "Used", render: (a) => fmtBytes(a.used) }, { label: "", render: (a) => usageBar(a.used, r.quota.bytes) }, ], q.addresses)), [{ label: "Close", value: null }], { wide: true }); diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js index 6bd1c01..e646d81 100644 --- a/internal/web/static/assets/pages/stats.js +++ b/internal/web/static/assets/pages/stats.js @@ -3,6 +3,7 @@ import { h, mount, fmtTime, errorBox } from "../ui.js"; import { get } from "../api.js"; import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js"; +import { withNames } from "../widgets.js"; const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"], ["365d", "a year"]]; const rangeSecs = { "1h": 3600, "6h": 6 * 3600, "24h": 86400, "7d": 7 * 86400, "30d": 30 * 86400, "90d": 90 * 86400, "365d": 365 * 86400 }; @@ -142,6 +143,7 @@ async function loadAll(since, heatSince) { get("/stats/timeline", { since, keys: "g:conntrack,g:conntrack_max,g:mem,g:rss,g:load,g:cpu" }), get("/stats/timeline", { since, keys: "quota:*" }), ]); + await withNames(summary.devices_down, summary.devices_up, summary.blocked_sources); return { summary, zones, traffic, drops, heat, hits, gauges, quotas: quotas.series.map((x) => ({ key: x.key.slice(6), count: x.values.reduce((a, v) => a + Math.max(v, 0), 0) })).filter((e) => e.count > 0).sort((a, b) => b.count - a.count) }; } diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index 7a11ecb..5f214d5 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -85,6 +85,7 @@ h1 { font-size: 20px; margin: 0 0 4px; } h2 { font-size: 16px; margin: 18px 0 8px; } h3 { font-size: 14px; margin: 14px 0 6px; } .muted { color: var(--muted); } +.host-name { font-weight: 600; } .small { font-size: 12px; } .nowrap { white-space: nowrap; } .warn-text { color: var(--warn); } diff --git a/internal/web/static/assets/widgets.js b/internal/web/static/assets/widgets.js index a35f2f4..c1f1f03 100644 --- a/internal/web/static/assets/widgets.js +++ b/internal/web/static/assets/widgets.js @@ -1,7 +1,7 @@ // Widgets shared by several pages: list editors with suggestions, the // match editor of rules and NAT, descriptions, hit counts, durations. import { h, input, select, field, checkbox } from "./ui.js"; -import { get } from "./api.js"; +import { get, api } from "./api.js"; // ---- durations @@ -163,6 +163,27 @@ export function usageBar(used, total) { return h("div", { class: "usage" }, h("div", { class: "usage-bar" + (used >= total ? " full" : "") }, seg)); } +// lookupNames asks for inside addresses' names (docs/configuration.md): +// {address: name}, {} when names are off or can't be had. +export async function lookupNames(addrs) { + const list = [...new Set(addrs.filter(Boolean))]; + if (!list.length) return {}; + try { return (await api("POST", "/names", { body: { addresses: list } })).names || {}; } catch { return {}; } +} + +// named shows an address (or text with it, such as address:port) after +// its name's first label; the whole name is in the tooltip. +export function named(names, addr, text = addr) { + const n = names[addr]; + return n ? h("span", { title: n + " (" + addr + ")" }, h("span", { class: "host-name" }, n.split(".")[0]), " ", h("span", { class: "muted" }, text)) : text; +} + +// withNames sets each top-list entry's name from its key (an address). +export async function withNames(...lists) { + const names = await lookupNames(lists.flat().map((e) => e.key)); + for (const e of lists.flat()) if (names[e.key]) e.name = names[e.key].split(".")[0]; +} + // quotas loads what the rules' quotas used, by rule ID ({} without // log:read). export async function quotas() { diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 8978bd1..2487bdc 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -55,6 +55,9 @@ docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls --network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1 docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1 docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1 +# The client has a name for housecarld's reverse lookups (/etc/hosts +# here, a PTR record in real life); the addresses outside get none. +docker exec ${P}-router sh -c "echo '$CLIENT4 e2e-client.lab' >> /etc/hosts" || exit 1 # The router's interfaces get the names of the real one: wan, enp2s0, wg0 # (attached in that order: eth0, eth1, eth2). docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done @@ -319,11 +322,11 @@ send 2000000 >/dev/null 2>&1 run "used up: the device is cut off" 1 send 1000 run "the usage" 0 H quotas contains "e2e quota.*1.0 MB each.*by hand" -contains "$CLIENT4 *1\.[0-9] MB" +contains "$CLIENT4 (e2e-client) *1\.[0-9] MB" stop; start run "after a restart, still used up" 1 send 1000 run "the usage after a restart" 0 H quotas -contains "$CLIENT4 *1\.[0-9] MB" +contains "$CLIENT4 (e2e-client) *1\.[0-9] MB" run "reset by hand" 0 H rule reset-quota "$QR" run "the device is back" 0 send 1000 run "the reset is audited" 0 H audit --action quota. @@ -362,7 +365,8 @@ contains "Dropped: *[1-9]" run "traffic per zone over time" 0 H stats traffic --since 1h --step 1m contains "wan ↓" run "the devices" 0 H stats top devices --since 1h -contains "^$CLIENT4 *[0-9.]* KB *[0-9.]* KB" +contains "^$CLIENT4 (e2e-client) *[0-9.]* KB *[0-9.]* KB" +run "each device once" 0 sh -c "docker exec ${P}-router /w/hc stats top devices --since 1h | grep -c '^$CLIENT4 ' | grep -qx 1" run "the sources blocked" 0 H stats top blocked --since 1h contains "^$NET4 *[1-9]" run "what they tried" 0 H stats top tried --since 1h @@ -379,6 +383,8 @@ contains "^[0-9-]* [0-9:]* *[0-9]* *[0-9]*" docker exec -d ${P}-client sh -c "sleep 30 | nc $NET4 9000" sleep 1 run "the connection is tracked" 0 H connections --search "$CLIENT4 :9000" +contains "$CLIENT4:[0-9]* (e2e-client) *$NET4:9000 *as $WAN4" +run "without names" 0 H connections --search "$CLIENT4 :9000" --no-names contains "$CLIENT4:[0-9]* *$NET4:9000 *as $WAN4" run "closed" 0 H connections close --search "$CLIENT4 :9000" -y contains "closed [1-9][0-9]* connection"