housecarl/internal/nft/nft.go

// Package nft runs the nft command: it checks and applies rendered
// rulesets, reads counters and dynamic sets back as JSON, and maps nft's
// error messages to the objects they came from.
//
// The nft command is used rather than a netlink library: it checks a
// ruleset without applying it (nft -c), applies a whole script in one
// transaction, and its errors are readable.
package nft

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"housecarl/internal/model"
	"housecarl/internal/render"
	"housecarl/internal/trace"
)

// Runner runs nft.
type Runner struct {
	// Command is how nft is run: ["nft"] (the default), or a wrapper such
	// as ["docker", "run", "--rm", "-i", "--cap-add", "NET_ADMIN", "IMAGE",
	// "nft"] in tests.
	Command []string
}

// Wrapper reads HOUSECARL_NFT, a command line that runs nft (tests point
// it at a container), or returns ["nft"].
func Wrapper() []string {
	if v := strings.TrimSpace(os.Getenv("HOUSECARL_NFT")); v != "" {
		return strings.Fields(v)
	}
	return []string{"nft"}
}

func (r *Runner) cmd(ctx context.Context, args ...string) *exec.Cmd {
	c := r.Command
	if len(c) == 0 {
		c = []string{"nft"}
	}
	return exec.CommandContext(ctx, c[0], append(append([]string{}, c[1:]...), args...)...)
}

// run runs nft with args, script on stdin.
func (r *Runner) run(ctx context.Context, script string, args ...string) ([]byte, error) {
	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()
	c := r.cmd(ctx, args...)
	if script != "" {
		c.Stdin = strings.NewReader(script)
	}
	var out, errb bytes.Buffer
	c.Stdout, c.Stderr = &out, &errb
	if err := c.Run(); err != nil {
		msg := strings.TrimSpace(errb.String())
		if msg == "" {
			msg = err.Error()
		}
		return out.Bytes(), &Error{Output: msg}
	}
	return out.Bytes(), nil
}

// Error is nft's complaint.
type Error struct {
	Output string
	// Object is what the first failing line came from, when known.
	Object string
	// Message is the first error line, without the file and position.
	Message string
}

func (e *Error) Error() string {
	if e.Object != "" && e.Message != "" {
		return e.Object + ": " + e.Message
	}
	if e.Message != "" {
		return e.Message
	}
	return e.Output
}

// errLine matches "/dev/stdin:12:3-10: Error: …".
var errLine = regexp.MustCompile(`(?m)^[^:\n]*:(\d+):[0-9-]+: Error: (.*)$`)

// explain fills in Object and Message from nft's output and the rendered
// ruleset.
func explain(err error, rs *render.Ruleset) error {
	e, ok := err.(*Error)
	if !ok {
		return err
	}
	if m := errLine.FindStringSubmatch(e.Output); m != nil {
		line, _ := strconv.Atoi(m[1])
		e.Message = strings.TrimSpace(m[2])
		e.Object = rs.Object(line)
	}
	return e
}

// Check has nft check a ruleset without applying it.
func (r *Runner) Check(ctx context.Context, rs *render.Ruleset) error {
	_, err := r.run(ctx, rs.Text, "-c", "-f", "/dev/stdin")
	return explain(err, rs)
}

// Apply applies a ruleset (one transaction: all or nothing).
func (r *Runner) Apply(ctx context.Context, rs *render.Ruleset) error {
	_, err := r.run(ctx, rs.Text, "-f", "/dev/stdin")
	return explain(err, rs)
}

// ListTable returns Housecarl's table as nft JSON (nil if it doesn't
// exist).
func (r *Runner) ListTable(ctx context.Context) ([]byte, error) {
	out, err := r.run(ctx, "", "-j", "list", "table", "inet", render.Table)
	if err != nil {
		if e, ok := err.(*Error); ok && strings.Contains(e.Output, "No such file or directory") {
			return nil, nil
		}
		return nil, err
	}
	return out, nil
}

// ListRuleset returns the whole ruleset as nft JSON (the importer's
// input on this machine).
func (r *Runner) ListRuleset(ctx context.Context) ([]byte, error) {
	return r.run(ctx, "", "-j", "list", "ruleset")
}

// TableText returns a table as an nft script that recreates it.
func (r *Runner) TableText(ctx context.Context, family, name string) (string, error) {
	out, err := r.run(ctx, "", "list", "table", family, name)
	if err != nil {
		return "", err
	}
	return "table " + family + " " + name + " {}\ndelete table " + family + " " + name + "\n" + string(out), nil
}

// ApplyScript applies an nft script (a takeover's rollback).
func (r *Runner) ApplyScript(ctx context.Context, script string) error {
	_, err := r.run(ctx, script, "-f", "/dev/stdin")
	return err
}

// DeleteTable deletes a table (hc takeover: the ruleset Housecarl
// replaces).
func (r *Runner) DeleteTable(ctx context.Context, family, name string) error {
	_, err := r.run(ctx, "", "delete", "table", family, name)
	return err
}

// AddElement adds an entry to a dynamic set's family part, with a timeout
// in seconds (0: the set's default).
func (r *Runner) AddElement(ctx context.Context, set, addr string, timeout int) error {
	a, err := model.ParseAddress(addr)
	if err != nil {
		return err
	}
	t := ""
	if timeout > 0 {
		t = fmt.Sprintf(" timeout %ds", timeout)
	}
	_, err = r.run(ctx, fmt.Sprintf("add element inet %s %s { %s%s }\n", render.Table, render.SetName(set, a.IsV6()), a, t), "-f", "/dev/stdin")
	return err
}

// DeleteElement removes an entry from a dynamic set.
func (r *Runner) DeleteElement(ctx context.Context, set, addr string) error {
	a, err := model.ParseAddress(addr)
	if err != nil {
		return err
	}
	_, err = r.run(ctx, fmt.Sprintf("delete element inet %s %s { %s }\n", render.Table, render.SetName(set, a.IsV6()), a), "-f", "/dev/stdin")
	return err
}

// ---- reading JSON

// Table is what Housecarl reads back from its table: counters by mark
// and the dynamic sets' entries.
type Table struct {
	// Counters by the comment's mark ("r12", "f3", "b-established",
	// "p-input"); several nftables rules of one object add up.
	Counters map[string]Counter
	// Elements by set name (both families together).
	Elements map[string][]render.Element
	// Quotas: the rules' quotas' used bytes (quota_r12), and the
	// per-address quotas' entries by set (quota_r12_v4).
	QuotaUsage
	// DeviceSets are the per-device traffic counters by set
	// (stats_up_v4, stats_down_v6 …): each address's packets and bytes.
	DeviceSets map[string][]render.Element
	// Since is when the counters started (housecarld's first apply).
	Since time.Time
	// Tables are all the machine's tables ("inet main"), when the whole
	// ruleset was read.
	Tables []string
	// Fallback: the table is the boot fallback (render.Fallback), not a
	// ruleset housecarld applied.
	Fallback bool
}

// QuotaUsage is what the quotas used: kept across reloads, and saved in
// the database so a reboot doesn't reset it.
type QuotaUsage struct {
	Quotas    map[string]int64            `json:"quotas,omitempty"`
	QuotaSets map[string][]render.Element `json:"quota_sets,omitempty"`
}

// merge takes the larger usage of each quota and entry from o (what was
// saved before a reboot, when the kernel's count started again).
func (u *QuotaUsage) merge(o *QuotaUsage) {
	if o == nil {
		return
	}
	if u.Quotas == nil {
		u.Quotas = map[string]int64{}
	}
	if u.QuotaSets == nil {
		u.QuotaSets = map[string][]render.Element{}
	}
	for k, v := range o.Quotas {
		if v > u.Quotas[k] {
			u.Quotas[k] = v
		}
	}
	for set, es := range o.QuotaSets {
		for _, e := range es {
			found := false
			for i := range u.QuotaSets[set] {
				if u.QuotaSets[set][i].Addr == e.Addr {
					found = true
					if e.Used > u.QuotaSets[set][i].Used {
						u.QuotaSets[set][i].Used = e.Used
					}
				}
			}
			if !found {
				u.QuotaSets[set] = append(u.QuotaSets[set], e)
			}
		}
	}
}

// Counter is packets and bytes.
type Counter struct {
	Packets uint64 `json:"packets"`
	Bytes   uint64 `json:"bytes"`
}

type jsonDoc struct {
	Nftables []map[string]json.RawMessage `json:"nftables"`
}

type jsonRule struct {
	Comment string            `json:"comment"`
	Expr    []json.RawMessage `json:"expr"`
}

type jsonSet struct {
	Name string            `json:"name"`
	Elem []json.RawMessage `json:"elem"`
}

// jsonQuota is a quota's state: a named quota ({"bytes", "used"}, in
// bytes) or one in an element ({"val", "val_unit", "used", "used_unit"}).
type jsonQuota struct {
	Name     string `json:"name"`
	Used     int64  `json:"used"`
	UsedUnit string `json:"used_unit"`
}

func (q jsonQuota) bytes() int64 {
	if m, ok := model.ByteUnits[q.UsedUnit]; ok {
		return q.Used * m
	}
	return q.Used
}

// ParseTable reads `nft -j list table inet housecarl`.
func ParseTable(data []byte) (*Table, error) {
	t := &Table{Counters: map[string]Counter{}, Elements: map[string][]render.Element{}, DeviceSets: map[string][]render.Element{},
		QuotaUsage: QuotaUsage{Quotas: map[string]int64{}, QuotaSets: map[string][]render.Element{}}}
	if len(data) == 0 {
		return t, nil
	}
	var doc jsonDoc
	if err := json.Unmarshal(data, &doc); err != nil {
		return nil, fmt.Errorf("reading nft's JSON: %w", err)
	}
	for _, item := range doc.Nftables {
		if raw, ok := item["rule"]; ok {
			var r jsonRule
			if json.Unmarshal(raw, &r) != nil || !strings.HasPrefix(r.Comment, "hc:") {
				continue
			}
			mark, _, _ := strings.Cut(strings.TrimPrefix(r.Comment, "hc:"), " ")
			if strings.HasPrefix(mark, render.FallbackMark+"-") {
				t.Fallback = true
			}
			for _, e := range r.Expr {
				var c struct {
					Counter *Counter `json:"counter"`
				}
				if json.Unmarshal(e, &c) == nil && c.Counter != nil {
					cur := t.Counters[mark]
					cur.Packets += c.Counter.Packets
					cur.Bytes += c.Counter.Bytes
					t.Counters[mark] = cur
				}
			}
		}
		if raw, ok := item["quota"]; ok {
			var q jsonQuota
			if json.Unmarshal(raw, &q) == nil && strings.HasPrefix(q.Name, "quota_r") {
				t.Quotas[q.Name] = q.bytes()
			}
		}
		if raw, ok := item["set"]; ok {
			var s jsonSet
			if json.Unmarshal(raw, &s) != nil {
				continue
			}
			if strings.HasPrefix(s.Name, "quota_r") {
				for _, e := range s.Elem {
					var w struct {
						Elem struct {
							Val   string     `json:"val"`
							Quota *jsonQuota `json:"quota"`
						} `json:"elem"`
					}
					if json.Unmarshal(e, &w) == nil && w.Elem.Quota != nil {
						t.QuotaSets[s.Name] = append(t.QuotaSets[s.Name], render.Element{Addr: w.Elem.Val, Used: w.Elem.Quota.bytes()})
					}
				}
				continue
			}
			if render.IsDeviceSet(s.Name) {
				for _, e := range s.Elem {
					var w struct {
						Elem struct {
							Val     string   `json:"val"`
							Expires int      `json:"expires"`
							Counter *Counter `json:"counter"`
						} `json:"elem"`
					}
					if json.Unmarshal(e, &w) == nil && w.Elem.Counter != nil {
						t.DeviceSets[s.Name] = append(t.DeviceSets[s.Name], render.Element{Addr: w.Elem.Val, Timeout: w.Elem.Expires, Packets: w.Elem.Counter.Packets, Bytes: w.Elem.Counter.Bytes})
					}
				}
				continue
			}
			name := strings.TrimSuffix(strings.TrimSuffix(s.Name, "_v4"), "_v6")
			if name == s.Name || strings.HasPrefix(s.Name, "meter_") || strings.HasPrefix(s.Name, "conn_") {
				continue
			}
			for _, e := range s.Elem {
				if el, ok := parseElem(e); ok {
					t.Elements[name] = append(t.Elements[name], el)
				}
			}
		}
	}
	return t, nil
}

// parseElem reads a set element: "10.0.0.5", or {"elem": {"val":
// "10.0.0.5", "timeout": 1800, "expires": 1799}}.
func parseElem(raw json.RawMessage) (render.Element, bool) {
	var s string
	if json.Unmarshal(raw, &s) == nil {
		return render.Element{Addr: s}, true
	}
	var w struct {
		Elem struct {
			Val     json.RawMessage `json:"val"`
			Expires int             `json:"expires"`
		} `json:"elem"`
	}
	if json.Unmarshal(raw, &w) != nil || json.Unmarshal(w.Elem.Val, &s) != nil {
		return render.Element{}, false
	}
	return render.Element{Addr: s, Timeout: w.Elem.Expires}, true
}

// ---- the firewall

// Firewall is the engine's Firewall: it renders configurations and has
// nft check and apply them, carrying dynamic sets' entries over.
type Firewall struct {
	Runner *Runner
	// Interfaces lists the interfaces that exist now (nil: assume all).
	Interfaces func() map[string]bool
	// BootFile is where the confirmed ruleset is saved for the boot unit
	// (housecarl-ruleset.service; "" for none).
	BootFile string
	// Shadow is shadow mode: the ruleset changes nothing, beside another
	// firewall (render.Options; docs/shadow.md). ShadowBefore runs its
	// filter chains before that one's.
	Shadow, ShadowBefore bool

	// Replacing the table resets its counters: base keeps the totals of
	// the tables replaced since housecarld started.
	mu    sync.Mutex
	base  map[string]Counter
	since time.Time
	// restore is the quota usage saved before housecarld (or the machine)
	// stopped, put back by the next apply.
	restore *QuotaUsage
	// flowDevs are the flowtable's devices in the ruleset in force, by
	// name, with their interface index then (0: left out, missing).
	flowDevs map[string]int
}

// ifIndex is an interface's index, 0 if it doesn't exist (tests change it).
var ifIndex = func(name string) int {
	if i, err := net.InterfaceByName(name); err == nil {
		return i.Index
	}
	return 0
}

// FlowtableStale reports whether the flowtable in force no longer matches
// the machine: a device of st's flowtable appeared, went, or came back as
// a new interface (ppp0 after PPPoE reconnects; the kernel drops a
// device from the flowtable when it goes). Applying again fixes it.
func (f *Firewall) FlowtableStale(st *model.State) bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.flowDevs == nil {
		return false
	}
	for _, d := range st.Settings.FlowtableDevices {
		if ifIndex(d) != f.flowDevs[d] {
			return true
		}
	}
	return false
}

// RestoreQuotas has the next apply put this saved quota usage back where
// the kernel's count is lower (after a reboot, the boot ruleset starts
// every quota at zero).
func (f *Firewall) RestoreQuotas(u *QuotaUsage) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.restore = u
}

// ResetQuota starts a rule's quota again: the named quota, or the
// per-address quota sets (both families).
func (f *Firewall) ResetQuota(ctx context.Context, ruleID int64, perAddress bool) error {
	name := render.QuotaName(ruleID)
	if !perAddress {
		_, err := f.Runner.run(ctx, "", "reset", "quota", "inet", render.Table, name)
		return err
	}
	for _, fam := range []string{"_v4", "_v6"} {
		if _, err := f.Runner.run(ctx, "", "flush", "set", "inet", render.Table, name+fam); err != nil {
			if e, ok := err.(*Error); ok && strings.Contains(e.Output, "No such file or directory") {
				continue // the rule has no such family
			}
			return err
		}
	}
	return nil
}

func (f *Firewall) options() render.Options {
	o := render.Options{Shadow: f.Shadow, ShadowBefore: f.ShadowBefore}
	if f.Interfaces != nil {
		o.Interfaces = f.Interfaces()
	}
	return o
}

// Render builds the ruleset for st as Apply would.
func (f *Firewall) Render(st *model.State) *render.Ruleset { return render.Render(st, f.options()) }

// RenderEnforcing builds the ruleset for st as Apply would outside shadow
// mode.
func (f *Firewall) RenderEnforcing(st *model.State) *render.Ruleset {
	o := f.options()
	o.Shadow, o.ShadowBefore = false, false
	return render.Render(st, o)
}

// Check implements engine.Firewall. In shadow mode it checks the
// enforcing ruleset too, so that leaving shadow mode can't fail.
func (f *Firewall) Check(ctx context.Context, st *model.State) error {
	if err := f.Runner.Check(ctx, render.Render(st, f.options())); err != nil {
		return err
	}
	if f.Shadow {
		if err := f.Runner.Check(ctx, f.RenderEnforcing(st)); err != nil {
			return fmt.Errorf("the ruleset for when shadow mode is off: %w", err)
		}
	}
	return nil
}

// Apply implements engine.Firewall.
func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	opt := f.options()
	var old *Table
	if data, err := f.Runner.ListTable(ctx); err == nil {
		if t, err := ParseTable(data); err == nil {
			opt.Elements = t.Elements
			old = t
		}
	}
	usage := QuotaUsage{}
	if old != nil {
		usage = old.QuotaUsage
	}
	usage.merge(f.restore)
	opt.Quotas, opt.QuotaSets = usage.Quotas, usage.QuotaSets
	if old != nil {
		opt.DeviceSets = old.DeviceSets
	}
	if err := f.Runner.Apply(ctx, render.Render(st, opt)); err != nil {
		return err
	}
	f.restore = nil
	f.flowDevs = map[string]int{}
	for _, d := range st.Settings.FlowtableDevices {
		if opt.Interfaces == nil || opt.Interfaces[d] {
			f.flowDevs[d] = ifIndex(d)
		}
	}
	if f.base == nil {
		f.base, f.since = map[string]Counter{}, time.Now()
	}
	if old != nil {
		for k, c := range old.Counters {
			b := f.base[k]
			b.Packets += c.Packets
			b.Bytes += c.Bytes
			f.base[k] = b
		}
	}
	pruneCounters(f.base, st)
	return nil
}

// pruneCounters forgets the totals of objects st no longer has (deleted,
// or replaced by an import with new IDs), so they aren't listed beside
// the current ones. A disabled object keeps its totals for when it comes
// back; built-ins and policies are always kept.
func pruneCounters(base map[string]Counter, st *model.State) {
	ids := map[string]bool{}
	for _, r := range st.Rules {
		ids[fmt.Sprintf("r%d", r.ID)] = true
	}
	for _, x := range st.Forwards {
		ids[fmt.Sprintf("f%d", x.ID)] = true
	}
	for _, x := range st.SNATs {
		ids[fmt.Sprintf("s%d", x.ID)] = true
	}
	for _, x := range st.Redirects {
		ids[fmt.Sprintf("d%d", x.ID)] = true
	}
	for mark := range base {
		obj, _, _ := strings.Cut(mark, "-") // f3-hairpin, f3-6 are f3's
		if len(obj) < 2 || !strings.ContainsRune("rfsd", rune(obj[0])) || strings.Trim(obj[1:], "0123456789") != "" {
			continue // b-established, p-input
		}
		if !ids[obj] {
			delete(base, mark)
		}
	}
}

// SaveBoot writes the ruleset of st for the boot unit, and the fallback
// it loads if that fails (FallbackFile), atomically. The boot ruleset has
// no flowtable: it loads before the network comes up, when its devices
// (ppp0, wg0) don't exist yet and nftables would refuse the whole file;
// housecarld adds it once they do. In shadow mode both are shadow mode's:
// the fallback then filters nothing.
func (f *Firewall) SaveBoot(st *model.State) error {
	if f.BootFile == "" {
		return nil
	}
	if err := os.MkdirAll(filepath.Dir(f.BootFile), 0o750); err != nil {
		return err
	}
	fallback := render.Fallback(st)
	if f.Shadow {
		fallback = render.ShadowFallback()
	}
	if err := writeAtomic(FallbackFile(f.BootFile), fallback); err != nil {
		return err
	}
	rs := render.Render(st, render.Options{Interfaces: map[string]bool{}, Shadow: f.Shadow, ShadowBefore: f.ShadowBefore})
	return writeAtomic(f.BootFile, rs.Text)
}

// FallbackFile is where the fallback for a boot ruleset is saved:
// ruleset.nft's is ruleset-fallback.nft beside it.
func FallbackFile(boot string) string {
	return strings.TrimSuffix(boot, ".nft") + "-fallback.nft"
}

func writeAtomic(path, text string) error {
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, []byte(text), 0o640); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// Table reads Housecarl's table back, with counters totalled since
// housecarld started (across reloads).
func (f *Firewall) Table(ctx context.Context) (*Table, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	data, err := f.Runner.ListTable(ctx)
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, ErrNoTable
	}
	t, err := ParseTable(data)
	if err != nil {
		return nil, err
	}
	for k, b := range f.base {
		c := t.Counters[k]
		c.Packets += b.Packets
		c.Bytes += b.Bytes
		t.Counters[k] = c
	}
	t.Since = f.since
	return t, nil
}

// ErrNoTable: Housecarl's table isn't in the ruleset.
var ErrNoTable = errors.New("there is no table inet housecarl: Housecarl's ruleset isn't in force")

// AddElement adds an entry to a dynamic set.
func (f *Firewall) AddElement(ctx context.Context, set, addr string, timeout int) error {
	return f.Runner.AddElement(ctx, set, addr, timeout)
}

// DeleteElement removes an entry from a dynamic set.
func (f *Firewall) DeleteElement(ctx context.Context, set, addr string) error {
	return f.Runner.DeleteElement(ctx, set, addr)
}

// ListRuleset returns the machine's whole ruleset as JSON.
func (f *Firewall) ListRuleset(ctx context.Context) ([]byte, error) { return f.Runner.ListRuleset(ctx) }

// TableText returns another table as a script that recreates it.
func (f *Firewall) TableText(ctx context.Context, family, name string) (string, error) {
	return f.Runner.TableText(ctx, family, name)
}

// ApplyScript applies an nft script.
func (f *Firewall) ApplyScript(ctx context.Context, script string) error {
	return f.Runner.ApplyScript(ctx, script)
}

// Trace follows packets: it starts `nft monitor trace`, puts the tracing
// table (script) in, and passes each line to line until ctx ends; then
// it takes the table out again, whatever happened.
func (f *Firewall) Trace(ctx context.Context, script string, line func(string)) error {
	c := f.Runner.cmd(ctx, "monitor", "trace")
	out, err := c.StdoutPipe()
	if err != nil {
		return err
	}
	var errb bytes.Buffer
	c.Stderr = &errb
	if err := c.Start(); err != nil {
		return err
	}
	defer f.Runner.run(context.Background(), "", "delete", "table", "inet", trace.Table)
	if err := f.Runner.ApplyScript(ctx, script); err != nil {
		c.Process.Kill()
		c.Wait()
		return fmt.Errorf("starting the trace: %w", err)
	}
	sc := bufio.NewScanner(out)
	sc.Buffer(make([]byte, 64<<10), 1<<20)
	for sc.Scan() {
		line(sc.Text())
	}
	err = c.Wait()
	if ctx.Err() != nil {
		return nil // stopped, or its time was up
	}
	if err != nil && errb.Len() > 0 {
		return fmt.Errorf("nft monitor trace: %s", strings.TrimSpace(errb.String()))
	}
	return err
}

// DeleteTable deletes another table (takeover).
func (f *Firewall) DeleteTable(ctx context.Context, family, name string) error {
	if name == render.Table && family == "inet" {
		return fmt.Errorf("that is Housecarl's own table")
	}
	return f.Runner.DeleteTable(ctx, family, name)
}