Connections, "every packet" rules, and a packet tracer
- Connections: the kernel's tracked connections, read and closed over ctnetlink (internal/conntrack, raw netlink like the drop log), with their NAT and TCP state; closing one makes the rules check its next packet again (GET /connections, POST /connections/close; hc connections; Traffic → Connections). - Rules can check every packet (every_packet, --every-packet): before established connections are accepted, so a drop or reject also stops connections already open. They can't accept. - A packet tracer (internal/trace): a table of its own, inet housecarl-trace, marks the packets a filter picks, `nft monitor trace` reports every rule and policy they meet in every table, and the tracer names what accepted or dropped them, Housecarl's rules by name (POST, GET, DELETE /trace; hc trace; Traffic → Trace). One trace at a time, at most 200 packets and 20 a second; the table goes when it ends. e2e: 157 checks, among them an upload a normal block doesn't stop and an every-packet one does, and a trace naming the blocking rule. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
abrown38andClaude Opus 5.5 committed
1 parent
98e11a8bec
commit
a45c893830
34 files changed
+1830
-41
No files matched your search
@@ -0,0 +1,258 @@
|
||||
// Package trace follows packets through the whole ruleset, every table's
|
||||
// chains, with nftables' own tracing: a table of its own (housecarl-trace,
|
||||
// hooked before everything else) marks the packets a filter picks
|
||||
// (meta nftrace set 1), and `nft monitor trace` reports each chain and
|
||||
// rule they pass, and what decided their fate. Packets are counted per
|
||||
// trace id: one packet through the hooks it crosses.
|
||||
package trace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"housecarl/internal/api"
|
||||
)
|
||||
|
||||
// Table is the tracing table's name (family inet).
|
||||
const Table = "housecarl-trace"
|
||||
|
||||
// Script builds the tracing table for a filter: packets from or to the
|
||||
// address, on the port (either way), of the protocol. A limit keeps a
|
||||
// flood from swamping the trace.
|
||||
func Script(in api.TraceInput) (string, error) {
|
||||
var addr netip.Prefix
|
||||
if in.Address != "" {
|
||||
p, err := netip.ParsePrefix(in.Address)
|
||||
if err != nil {
|
||||
a, err2 := netip.ParseAddr(in.Address)
|
||||
if err2 != nil {
|
||||
return "", fmt.Errorf("address %q: give an address or a prefix (192.168.1.50, 2001:db8::/64)", in.Address)
|
||||
}
|
||||
p = netip.PrefixFrom(a, a.BitLen())
|
||||
}
|
||||
addr = p.Masked()
|
||||
}
|
||||
if in.Port < 0 || in.Port > 65535 {
|
||||
return "", fmt.Errorf("port %d: 1 to 65535", in.Port)
|
||||
}
|
||||
proto := strings.ToLower(in.Protocol)
|
||||
switch proto {
|
||||
case "", "tcp", "udp", "sctp", "icmp", "icmpv6":
|
||||
default:
|
||||
return "", fmt.Errorf("protocol %q: tcp, udp, sctp, icmp, icmpv6, or nothing for any", in.Protocol)
|
||||
}
|
||||
if !addr.IsValid() && in.Port == 0 {
|
||||
return "", fmt.Errorf("say what to trace: an address, a port, or both")
|
||||
}
|
||||
if in.Port != 0 && (proto == "icmp" || proto == "icmpv6") {
|
||||
return "", fmt.Errorf("ICMP has no ports")
|
||||
}
|
||||
var l4 string
|
||||
switch {
|
||||
case proto != "":
|
||||
l4 = "meta l4proto " + proto
|
||||
case in.Port != 0:
|
||||
l4 = "meta l4proto { tcp, udp, sctp }"
|
||||
}
|
||||
addrs := []string{""}
|
||||
if addr.IsValid() {
|
||||
ip := "ip"
|
||||
if addr.Addr().Is6() {
|
||||
ip = "ip6"
|
||||
}
|
||||
addrs = []string{ip + " saddr " + addr.String(), ip + " daddr " + addr.String()}
|
||||
}
|
||||
ports := []string{""}
|
||||
if in.Port != 0 {
|
||||
ports = []string{fmt.Sprintf("th dport %d", in.Port), fmt.Sprintf("th sport %d", in.Port)}
|
||||
}
|
||||
var rules []string
|
||||
for _, a := range addrs {
|
||||
for _, p := range ports {
|
||||
parts := []string{}
|
||||
for _, x := range []string{a, l4, p, "limit rate 20/second", "meta nftrace set 1"} {
|
||||
if x != "" {
|
||||
parts = append(parts, x)
|
||||
}
|
||||
}
|
||||
rules = append(rules, "\t\t"+strings.Join(parts, " "))
|
||||
}
|
||||
}
|
||||
body := strings.Join(rules, "\n")
|
||||
return fmt.Sprintf(`table inet %[1]s {}
|
||||
delete table inet %[1]s
|
||||
table inet %[1]s {
|
||||
chain pre {
|
||||
type filter hook prerouting priority -301; policy accept;
|
||||
%[2]s
|
||||
}
|
||||
chain out {
|
||||
type filter hook output priority -301; policy accept;
|
||||
%[2]s
|
||||
}
|
||||
}
|
||||
`, Table, body), nil
|
||||
}
|
||||
|
||||
// ---- reading `nft monitor trace`
|
||||
|
||||
// "trace id 38ff3662 inet housecarl input rule … (verdict accept)"
|
||||
var lineRe = regexp.MustCompile(`^trace id (\S+) (\S+) (\S+) (\S+) (packet: (.*)|rule (.*) \(verdict ([^)]*)\)|policy (\S+)|(.*))$`)
|
||||
|
||||
var (
|
||||
fieldRe = regexp.MustCompile(`\b(iif|oif) "([^"]*)"`)
|
||||
addrRe = regexp.MustCompile(`\b(ip6?) (saddr|daddr) (\S+)`)
|
||||
portRe = regexp.MustCompile(`\b(tcp|udp|sctp) (sport|dport) (\d+)`)
|
||||
flagsRe = regexp.MustCompile(`\btcp flags == (\S+)`)
|
||||
icmpRe = regexp.MustCompile(`\b(icmp|icmpv6) type (\S+)`)
|
||||
protoRe = regexp.MustCompile(`\bmeta l4proto (\S+)|\bip protocol (\S+)|\bip6 nexthdr (\S+)`)
|
||||
markRe = regexp.MustCompile(`comment "hc:(\S+)`)
|
||||
)
|
||||
|
||||
// Tracer collects packets from trace lines.
|
||||
type Tracer struct {
|
||||
// Max is how many packets it keeps (0: 200).
|
||||
Max int
|
||||
Now func() time.Time
|
||||
packets []*api.TracePacket
|
||||
byID map[string]*api.TracePacket
|
||||
// seen are the chains a packet entered: the kernel reuses trace ids
|
||||
// (they come from the packet's buffer), so entering one again means
|
||||
// another packet.
|
||||
seen map[*api.TracePacket]map[string]bool
|
||||
}
|
||||
|
||||
// Packets returns what was traced so far, oldest first.
|
||||
func (t *Tracer) Packets() []api.TracePacket {
|
||||
out := make([]api.TracePacket, 0, len(t.packets))
|
||||
for _, p := range t.packets {
|
||||
out = append(out, *p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Full reports whether it keeps no more packets.
|
||||
func (t *Tracer) Full() bool {
|
||||
max := t.Max
|
||||
if max == 0 {
|
||||
max = 200
|
||||
}
|
||||
return len(t.packets) >= max
|
||||
}
|
||||
|
||||
// Line reads one line of `nft monitor trace`. The tracing table's own
|
||||
// steps are left out.
|
||||
func (t *Tracer) Line(line string) {
|
||||
m := lineRe.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
id, table, chain := m[1], m[2]+" "+m[3], m[4]
|
||||
if m[3] == Table {
|
||||
return
|
||||
}
|
||||
if t.byID == nil {
|
||||
t.byID, t.seen = map[string]*api.TracePacket{}, map[*api.TracePacket]map[string]bool{}
|
||||
}
|
||||
p := t.byID[id]
|
||||
isPacket := m[6] != "" || strings.HasPrefix(m[5], "packet:")
|
||||
if p != nil && isPacket && t.seen[p][table+" "+chain] {
|
||||
p = nil // the same id, another packet
|
||||
}
|
||||
if p == nil {
|
||||
if t.Full() {
|
||||
return
|
||||
}
|
||||
now := time.Now
|
||||
if t.Now != nil {
|
||||
now = t.Now
|
||||
}
|
||||
p = &api.TracePacket{ID: id, Time: now(), Verdict: "accepted"}
|
||||
t.byID[id] = p
|
||||
t.seen[p] = map[string]bool{}
|
||||
t.packets = append(t.packets, p)
|
||||
}
|
||||
switch {
|
||||
case isPacket:
|
||||
t.seen[p][table+" "+chain] = true
|
||||
readPacket(p, m[6])
|
||||
case m[7] != "":
|
||||
step := api.TraceStep{Table: table, Chain: chain, Kind: "rule", Rule: m[7], Verdict: m[8]}
|
||||
if mk := markRe.FindStringSubmatch(m[7]); mk != nil {
|
||||
step.Mark = strings.TrimSuffix(mk[1], `"`)
|
||||
}
|
||||
p.Steps = append(p.Steps, step)
|
||||
decide(p, step)
|
||||
case m[9] != "":
|
||||
step := api.TraceStep{Table: table, Chain: chain, Kind: "policy", Verdict: m[9]}
|
||||
p.Steps = append(p.Steps, step)
|
||||
decide(p, step)
|
||||
}
|
||||
}
|
||||
|
||||
// decide notes a packet's fate: the first drop (or reject) ends it.
|
||||
func decide(p *api.TracePacket, s api.TraceStep) {
|
||||
if p.Verdict == "dropped" {
|
||||
return
|
||||
}
|
||||
switch strings.Fields(s.Verdict + " ")[0] {
|
||||
case "drop", "reject":
|
||||
p.Verdict, p.DecidedBy = "dropped", s
|
||||
case "accept":
|
||||
// What let it through is a filter chain's accept: Housecarl's
|
||||
// input, forward or output (NAT chains accept everything they
|
||||
// see). Another table's accept counts when Housecarl's says
|
||||
// nothing.
|
||||
filter := s.Table == "inet housecarl" && (s.Chain == "input" || s.Chain == "forward" || s.Chain == "output")
|
||||
if filter || p.DecidedBy.Verdict == "" && s.Table != "inet housecarl" {
|
||||
p.DecidedBy = s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readPacket fills in a packet's headers from its "packet:" line (once:
|
||||
// the first hook's view, before any NAT).
|
||||
func readPacket(p *api.TracePacket, f string) {
|
||||
for _, m := range fieldRe.FindAllStringSubmatch(f, -1) {
|
||||
if m[1] == "iif" && p.In == "" {
|
||||
p.In = m[2]
|
||||
}
|
||||
if m[1] == "oif" && p.Out == "" {
|
||||
p.Out = m[2]
|
||||
}
|
||||
}
|
||||
if p.Source != "" {
|
||||
return
|
||||
}
|
||||
for _, m := range addrRe.FindAllStringSubmatch(f, -1) {
|
||||
if m[2] == "saddr" {
|
||||
p.Source = m[3]
|
||||
} else {
|
||||
p.Destination = m[3]
|
||||
}
|
||||
}
|
||||
for _, m := range portRe.FindAllStringSubmatch(f, -1) {
|
||||
p.Protocol = m[1]
|
||||
n, _ := strconv.Atoi(m[3])
|
||||
if m[2] == "sport" {
|
||||
p.SourcePort = n
|
||||
} else {
|
||||
p.DestinationPort = n
|
||||
}
|
||||
}
|
||||
if m := flagsRe.FindStringSubmatch(f); m != nil {
|
||||
p.TCPFlags = m[1]
|
||||
}
|
||||
if m := icmpRe.FindStringSubmatch(f); m != nil {
|
||||
p.Protocol, p.ICMPType = m[1], m[2]
|
||||
}
|
||||
if p.Protocol == "" {
|
||||
if m := protoRe.FindStringSubmatch(f); m != nil {
|
||||
p.Protocol = strings.TrimSpace(m[1] + m[2] + m[3])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"housecarl/internal/api"
|
||||
)
|
||||
|
||||
// Lines nft 1.1.3's `nft monitor trace` printed for a connection to a
|
||||
// blocked port, and for one that loopback accepted.
|
||||
const sample = `trace id 38ff3662 inet housecarl-trace pre packet: iif "wan" ip saddr 198.51.100.7 ip daddr 203.0.113.10 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id 44978 ip protocol 6 ip length 60 tcp sport 43168 tcp dport 9999 tcp flags == syn tcp window 65495
|
||||
trace id 38ff3662 inet housecarl-trace pre rule ip saddr 198.51.100.7 meta nftrace set 1 (verdict continue)
|
||||
trace id 38ff3662 inet housecarl-trace pre policy accept
|
||||
trace id 38ff3662 inet housecarl input packet: iif "wan" ip saddr 198.51.100.7 ip daddr 203.0.113.10 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id 44978 ip protocol 6 ip length 60 tcp sport 43168 tcp dport 9999 tcp flags == syn tcp window 65495
|
||||
trace id 38ff3662 inet housecarl input rule ct state established,related counter packets 0 bytes 0 accept comment "hc:b-established" (verdict continue)
|
||||
trace id 38ff3662 inet housecarl input rule tcp dport 9999 counter packets 1 bytes 60 drop comment "hc:r12 block 9999" (verdict drop)
|
||||
trace id a232d9a0 inet housecarl output packet: oif "lo" ip saddr 127.0.0.1 ip daddr 127.0.0.1 ip protocol 6 tcp sport 43168 tcp dport 22 tcp flags == syn
|
||||
trace id a232d9a0 inet housecarl output policy accept
|
||||
trace id a232d9a0 ip other input rule ip saddr 127.0.0.1 accept (verdict accept)
|
||||
trace id 77 inet housecarl forward packet: iif "lan" oif "wan" ip saddr 192.168.1.20 ip daddr 198.51.100.10 tcp sport 3500 tcp dport 7000 tcp flags == syn
|
||||
trace id 77 inet housecarl forward rule ip saddr @trusted_v4 counter accept comment "hc:r40 trusted" (verdict accept)
|
||||
trace id 77 inet housecarl srcnat rule oifname "wan" counter masquerade comment "hc:s91" (verdict accept)`
|
||||
|
||||
func TestTracer(t *testing.T) {
|
||||
var tr Tracer
|
||||
for _, l := range strings.Split(sample, "\n") {
|
||||
tr.Line(l)
|
||||
}
|
||||
ps := tr.Packets()
|
||||
if len(ps) != 3 {
|
||||
t.Fatalf("packets: %+v", ps)
|
||||
}
|
||||
p := ps[0]
|
||||
if p.In != "wan" || p.Source != "198.51.100.7" || p.DestinationPort != 9999 || p.Protocol != "tcp" || p.TCPFlags != "syn" {
|
||||
t.Errorf("headers: %+v", p)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Verdict != "dropped" || p.DecidedBy.Mark != "r12" || p.DecidedBy.Chain != "input" {
|
||||
t.Errorf("steps: %+v", p)
|
||||
}
|
||||
if q := ps[1]; q.Verdict != "accepted" || q.Out != "lo" || len(q.Steps) != 2 || q.DecidedBy.Kind != "policy" {
|
||||
t.Errorf("second: %+v", q)
|
||||
}
|
||||
// Accepted by the forward rule, not by source NAT, which came after.
|
||||
if q := ps[2]; q.Verdict != "accepted" || q.DecidedBy.Mark != "r40" {
|
||||
t.Errorf("third: decided by %+v", q.DecidedBy)
|
||||
}
|
||||
}
|
||||
|
||||
// The kernel reuses trace ids: a connection's next packet can come with
|
||||
// the same one. Entering a chain again starts another packet.
|
||||
func TestReusedID(t *testing.T) {
|
||||
var tr Tracer
|
||||
for _, l := range strings.Split(`trace id 5 inet housecarl output packet: oif "lo" ip saddr 127.0.0.1 ip daddr 127.0.0.1 tcp sport 49078 tcp dport 8070 tcp flags == syn
|
||||
trace id 5 inet housecarl output policy accept
|
||||
trace id 5 inet housecarl srcnat packet: oif "lo" ip saddr 127.0.0.1 ip daddr 127.0.0.1 tcp sport 49078 tcp dport 8070 tcp flags == syn
|
||||
trace id 5 inet housecarl srcnat policy accept
|
||||
trace id 5 inet housecarl output packet: oif "lo" ip saddr 127.0.0.1 ip daddr 127.0.0.1 tcp sport 49078 tcp dport 8070 tcp flags == ack
|
||||
trace id 5 inet housecarl output policy accept`, "\n") {
|
||||
tr.Line(l)
|
||||
}
|
||||
ps := tr.Packets()
|
||||
if len(ps) != 2 || len(ps[0].Steps) != 2 || len(ps[1].Steps) != 1 || ps[1].TCPFlags != "ack" {
|
||||
t.Errorf("packets: %+v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScript(t *testing.T) {
|
||||
s, err := Script(api.TraceInput{Address: "192.168.1.50", Port: 443})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"table inet housecarl-trace {}\ndelete table inet housecarl-trace\n",
|
||||
"type filter hook prerouting priority -301; policy accept;",
|
||||
"ip saddr 192.168.1.50/32 meta l4proto { tcp, udp, sctp } th dport 443 limit rate 20/second meta nftrace set 1",
|
||||
"ip daddr 192.168.1.50/32 meta l4proto { tcp, udp, sctp } th sport 443 limit rate 20/second meta nftrace set 1",
|
||||
} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("missing %q in\n%s", want, s)
|
||||
}
|
||||
}
|
||||
if _, err := Script(api.TraceInput{}); err == nil {
|
||||
t.Error("tracing everything")
|
||||
}
|
||||
if _, err := Script(api.TraceInput{Port: 22, Protocol: "icmp"}); err == nil {
|
||||
t.Error("ICMP with a port")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user