Boot fallback, flowtable devices that come and go, healthz, renames

The boot unit fails closed: if ruleset.nft doesn't load, it loads
ruleset-fallback.nft (render.Fallback), saved beside it from the same
configuration: nothing new in from external zones, established, the
ICMPv6 IPv6 needs and DHCPv6 replies let in, the inside left alone and
masqueraded, and the unit marked failed with a critical message.
housecarld replaces it when it applies; status, the dashboard and
/healthz say while it is in force. no_external_zone now also says the
fallback closes everything without an external zone.

The boot ruleset has no flowtable: nftables on Debian 13's kernel
refuses one naming a device that doesn't exist yet (ppp0 before PPPoE),
and with it the whole file. internal/linkwatch listens to rtnetlink link
messages; housecarld applies again when a flowtable device appears or
comes back as a new interface (by ifindex), under the engine's lock.
docs/flowtable.md: what it does, when to use it and when not, how.

GET /healthz reports firewall (applied, unconfigured, fallback,
missing), changes on probation and the machine's warnings, with 503
unless in force, read at most every five seconds.

Renaming a zone, set or service changes what names it in the same
change (one history entry, one undo); dynamic sets keep their names.
The importer keeps a chain's head in order: once a would-be built-in
stays a rule, the ones after it do too, so the maintainer's ICMPv6
accepts stay behind the invalid drop as written. hc settings set
--flowtable none empties the list.

make test (nft 1.1.3) and make e2e (242 checks, with VLAN
sub-interfaces, the fallback and the flowtable) pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
abrown38andClaude Opus 5.5 committed 2026-10-19 23:16:46 +01:00
1 parent 835a9ba2cc
commit 9d766b9823
38 files changed
+1335 -84

No files matched your search

+6 -2
View File
@@ -244,12 +244,16 @@ func newProblems(before, after []validate.Problem) []validate.Problem {
return out
}
// ApplyCurrent puts the current configuration in force (at start).
// ApplyCurrent puts the current configuration in force again: at start,
// and when a flowtable device appears. It holds the engine's lock, so a
// change can't be applied in between.
func (e *Engine) ApplyCurrent(ctx context.Context) error {
if e.Firewall == nil {
return nil
}
return e.Firewall.Apply(ctx, e.State())
e.mu.Lock()
defer e.mu.Unlock()
return e.Firewall.Apply(ctx, e.state.Clone())
}
// Revert undoes a history entry, if the objects it changed are still as
+8
View File
@@ -63,6 +63,14 @@ func (h *Host) Up(name string) bool {
return h.up[name]
}
// Forget drops what was read, so the next question reads the interfaces
// again (after the kernel said one appeared or went).
func (h *Host) Forget() {
h.mu.Lock()
h.at = time.Time{}
h.mu.Unlock()
}
// Exists returns the set of interface names (nil if they can't be read).
func (h *Host) Exists() map[string]bool {
h.mu.Lock()
+64 -47
View File
@@ -940,8 +940,10 @@ func (im *importer) run() {
order := map[string]int{"dstnat": 0, "srcnat": 1, model.ChainEarly: 2, model.ChainInput: 3, model.ChainForward: 4, model.ChainOutput: 5}
sort.SliceStable(bases, func(i, j int) bool { return order[bases[i].kind] < order[bases[j].kind] })
im.st.Settings.Builtins = model.Builtins{}
found := map[string]map[string]string{} // builtin → chain → "prefix" or "later"
var deferred []func()
found := map[string]map[string]string{} // builtin → chain → "prefix", "later" or "demoted"
heads := map[string][]headRule{} // chain → the built-ins its rules start with, in order
var chains []string
rest := map[string][]*parsed{} // chain → the rules after its head
for _, b := range bases {
rules := im.flatten(b.name, nil, 0)
switch b.kind {
@@ -954,9 +956,9 @@ func (im *importer) run() {
im.srcnat(p)
}
default:
chains = append(chains, b.kind)
prefix := true
for _, p := range rules {
p := p
if bi := im.builtinOf(b.kind, p); bi != "" {
if found[bi] == nil {
found[bi] = map[string]string{}
@@ -967,18 +969,25 @@ func (im *importer) run() {
}
found[bi][b.kind] = where
if prefix {
heads[b.kind] = append(heads[b.kind], headRule{bi, p})
continue
}
}
prefix = false
kind := b.kind
deferred = append(deferred, func() { im.rule(kind, p) })
rest[b.kind] = append(rest[b.kind], p)
}
}
}
im.settleBuiltins(found)
for _, f := range deferred {
f()
im.settleBuiltins(found, heads)
for _, chain := range chains {
for _, h := range heads[chain] {
if found[h.builtin][chain] == "demoted" {
im.rule(chain, h.p)
}
}
for _, p := range rest[chain] {
im.rule(chain, p)
}
}
if len(im.flow) > 0 {
im.st.Settings.FlowtableDevices = im.flow[0].Dev
@@ -1239,10 +1248,20 @@ func (im *importer) builtinOf(chain string, p *parsed) string {
return ""
}
// headRule is a rule at the head of a chain that is a built-in.
type headRule struct {
builtin string
p *parsed
}
// settleBuiltins turns on the built-ins found where they go, in every
// chain they apply to; the others stay off, and the rules that looked
// like them are kept as rules.
func (im *importer) settleBuiltins(found map[string]map[string]string) {
// like them are kept as rules. Built-ins come before every rule, so once
// one at a chain's head stays a rule, the ones after it there stay rules
// too ("demoted"), in their order: an accept moved ahead of a drop (ICMPv6
// ahead of dropping invalid packets) would let through what the drop
// stopped.
func (im *importer) settleBuiltins(found map[string]map[string]string, heads map[string][]headRule) {
b := &im.st.Settings.Builtins
on := func(name string, chains ...string) bool {
for _, c := range chains {
@@ -1252,14 +1271,36 @@ func (im *importer) settleBuiltins(found map[string]map[string]string) {
}
return true
}
b.Loopback = on("loopback", model.ChainInput)
b.Established = on("established", model.ChainInput, model.ChainForward)
b.DropInvalid = on("invalid", model.ChainInput, model.ChainForward)
b.ICMPv6 = on("icmpv6", model.ChainInput) && on("icmpv6-nd", model.ChainInput)
b.PingInternal = on("ping", model.ChainInput)
b.DHCPv6Client = on("dhcpv6-client", model.ChainInput)
b.MSSClamp = on("mss-clamp", model.ChainForward)
b.Antispoof = on("antispoof", model.ChainEarly)
for {
b.Loopback = on("loopback", model.ChainInput)
b.Established = on("established", model.ChainInput, model.ChainForward)
b.DropInvalid = on("invalid", model.ChainInput, model.ChainForward)
b.ICMPv6 = on("icmpv6", model.ChainInput) && on("icmpv6-nd", model.ChainInput)
b.PingInternal = on("ping", model.ChainInput)
b.DHCPv6Client = on("dhcpv6-client", model.ChainInput)
b.MSSClamp = on("mss-clamp", model.ChainForward)
b.Antispoof = on("antispoof", model.ChainEarly)
kept := map[string]bool{"loopback": b.Loopback, "established": b.Established, "invalid": b.DropInvalid, "icmpv6": b.ICMPv6, "icmpv6-nd": b.ICMPv6,
"ping": b.PingInternal, "dhcpv6-client": b.DHCPv6Client, "mss-clamp": b.MSSClamp, "antispoof": b.Antispoof, "flowtable": true}
changed := false
for chain, hs := range heads {
cut := false
for _, h := range hs {
// mss-clamp and flowtable decide nothing: their place
// doesn't matter.
if !kept[h.builtin] && h.builtin != "mss-clamp" && h.builtin != "flowtable" {
cut = true
}
if cut && found[h.builtin][chain] == "prefix" {
found[h.builtin][chain] = "demoted"
changed = true
}
}
}
if !changed {
break
}
}
if !b.Antispoof {
im.st.Settings.AntispoofV4, im.st.Settings.AntispoofV6, im.st.Settings.OwnPrefixes = nil, nil, nil
} else if im.st.Settings.AntispoofV6 == nil {
@@ -1268,40 +1309,16 @@ func (im *importer) settleBuiltins(found map[string]map[string]string) {
if b.ICMPv6 {
im.note("the ICMPv6 built-in also accepts MLD from link-local addresses, and lets ICMPv6 errors through the forward chain (RFC 4890)")
}
// The ones found in one chain's head but not another's: explicit rules
// in the chains where they were found.
for name, chains := range found {
for chain, where := range chains {
if where != "prefix" {
continue
}
kept := map[string]bool{"loopback": b.Loopback, "established": b.Established, "invalid": b.DropInvalid, "icmpv6": b.ICMPv6, "icmpv6-nd": b.ICMPv6,
"ping": b.PingInternal, "dhcpv6-client": b.DHCPv6Client, "mss-clamp": b.MSSClamp, "antispoof": b.Antispoof, "flowtable": true}[name]
if !kept {
im.note("%s in the %s chain stays a rule (it isn't in every chain the built-in covers, or comes after other rules)", name, chain)
im.builtinRule(name, chain)
// The ones that stay rules, in their place: say which.
for _, chain := range []string{model.ChainEarly, model.ChainInput, model.ChainForward, model.ChainOutput} {
for _, h := range heads[chain] {
if found[h.builtin][chain] == "demoted" {
im.note("%s in the %s chain stays a rule, in its place (the built-in isn't in every chain it covers, or a rule before it stays a rule)", h.builtin, chain)
}
}
}
}
// builtinRule writes a built-in that stays off as a rule of its chain.
func (im *importer) builtinRule(name, chain string) {
r := &model.Rule{ID: im.newID(), Chain: chain, Position: im.nextPos(chain) - 1000}
switch name {
case "established":
r.CtState, r.Action = []string{"established", "related"}, model.ActionAccept
case "invalid":
r.CtState, r.Action = []string{"invalid"}, model.ActionDrop
case "loopback":
r.Raw, r.Action = `iifname "lo" accept`, model.ActionAccept
default:
im.note("built-in %s of the %s chain couldn't be kept; add it by hand", name, chain)
return
}
im.st.Rules = append(im.st.Rules, r)
}
func (im *importer) nextPos(chain string) float64 {
im.positions[chain]++
return im.positions[chain]
+15 -2
View File
@@ -50,11 +50,24 @@ func TestImportRouter(t *testing.T) {
if s := st.Set("masq_to_internet"); s == nil {
t.Errorf("no set masq_to_internet (sets %v)", setNames(st))
}
// Built-ins: the ones at the head of their chains.
// Built-ins: the ones at the head of their chains, up to the invalid
// drop, which stays a rule (below): the ICMP, ICMPv6 and DHCPv6
// accepts after it stay rules too, in their order, so nothing the
// drop stopped gets through ahead of it.
b := st.Settings.Builtins
if !b.Loopback || !b.Established || !b.ICMPv6 || !b.PingInternal || !b.DHCPv6Client || !b.MSSClamp || !b.Antispoof {
if !b.Loopback || !b.Established || b.ICMPv6 || b.PingInternal || b.DHCPv6Client || !b.MSSClamp || !b.Antispoof {
t.Errorf("built-ins %+v", b)
}
var input []string
for _, r := range st.Rules {
if r.Chain == model.ChainInput {
input = append(input, strings.Join(r.CtState, ",")+"|"+strings.Join(r.Services, ",")+"|"+r.Raw)
}
}
if len(input) < 5 || input[0] != "invalid||" || !strings.HasPrefix(input[1], "|icmp/echo-reply,icmp/echo-request|") ||
!strings.Contains(input[2], "icmpv6/packet-too-big") || !strings.Contains(input[3], "ip6 hoplimit 255") || !strings.Contains(input[4], "udp/546") {
t.Errorf("input rules start %q", input)
}
// The forward chain's asymmetric-routing rule sits between the
// established and invalid rules: invalid stays rules, in order.
if b.DropInvalid {
+96
View File
@@ -0,0 +1,96 @@
// Package linkwatch tells when the machine's network interfaces change:
// one appears, goes, or comes back (ppp0 when PPPoE reconnects). It
// listens to the kernel's rtnetlink link messages and calls back once
// they have been quiet for a moment, since one change brings several.
package linkwatch
import (
"context"
"encoding/binary"
"errors"
"fmt"
"os"
"time"
"golang.org/x/sys/unix"
)
// Settle is how long the link messages must be quiet before the callback.
const Settle = 2 * time.Second
// Watch calls changed after interfaces changed, until ctx ends. It returns
// an error only if it can't listen at all.
func Watch(ctx context.Context, changed func()) error {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC|unix.SOCK_NONBLOCK, unix.NETLINK_ROUTE)
if err != nil {
return fmt.Errorf("opening a netlink socket: %w", err)
}
if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK, Groups: unix.RTMGRP_LINK}); err != nil {
unix.Close(fd)
return fmt.Errorf("listening to link changes: %w", err)
}
// A non-blocking descriptor goes through Go's poller, so closing the
// file ends a Read in progress.
f := os.NewFile(uintptr(fd), "rtnetlink")
go func() {
<-ctx.Done()
f.Close()
}()
events := make(chan struct{}, 1)
go func() {
buf := make([]byte, 64<<10)
for {
n, err := f.Read(buf)
switch {
case err != nil && ctx.Err() != nil:
close(events)
return
case errors.Is(err, unix.ENOBUFS):
// Messages were lost: something changed anyway.
case err != nil:
time.Sleep(time.Second)
continue
case !links(buf[:n]):
continue
}
select {
case events <- struct{}{}:
default:
}
}
}()
var timer <-chan time.Time
for {
select {
case _, ok := <-events:
if !ok {
return nil
}
timer = time.After(Settle)
case <-timer:
timer = nil
changed()
}
}
}
// links reports whether a netlink datagram holds a link being added,
// changed or deleted.
func links(b []byte) bool {
for len(b) >= unix.NLMSG_HDRLEN {
size := int(binary.NativeEndian.Uint32(b[0:4]))
typ := binary.NativeEndian.Uint16(b[4:6])
if size < unix.NLMSG_HDRLEN || size > len(b) {
return false
}
if typ == unix.RTM_NEWLINK || typ == unix.RTM_DELLINK {
return true
}
size = (size + unix.NLMSG_ALIGNTO - 1) &^ (unix.NLMSG_ALIGNTO - 1)
if size >= len(b) {
return false
}
b = b[size:]
}
return false
}
+36
View File
@@ -0,0 +1,36 @@
package linkwatch
import (
"encoding/binary"
"testing"
"golang.org/x/sys/unix"
)
// msg is a netlink message of a type, with a payload of n bytes.
func msg(typ uint16, n int) []byte {
b := make([]byte, unix.NLMSG_HDRLEN+n)
binary.NativeEndian.PutUint32(b[0:4], uint32(len(b)))
binary.NativeEndian.PutUint16(b[4:6], typ)
return b
}
func TestLinks(t *testing.T) {
for _, c := range []struct {
name string
b []byte
want bool
}{
{"a new link", msg(unix.RTM_NEWLINK, 16), true},
{"a link going", msg(unix.RTM_DELLINK, 16), true},
{"an address", msg(unix.RTM_NEWADDR, 16), false},
{"a link after an address (padded)", append(msg(unix.RTM_NEWADDR, 13), append(make([]byte, 3), msg(unix.RTM_NEWLINK, 8)...)...), true},
{"truncated", msg(unix.RTM_NEWLINK, 16)[:10], false},
{"a length beyond the datagram", msg(unix.RTM_NEWLINK, 16)[:unix.NLMSG_HDRLEN+4], false},
{"empty", nil, false},
} {
if got := links(c.b); got != c.want {
t.Errorf("%s: %v", c.name, got)
}
}
}
+64 -5
View File
@@ -14,6 +14,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
@@ -209,6 +210,9 @@ type Table struct {
// Tables are all the machine's tables ("inet main"), when the whole
// ruleset was read.
Tables []string
// Fallback: the table is the boot fallback (render.Fallback), not a
// ruleset housecarld applied.
Fallback bool
}
// QuotaUsage is what the quotas used: kept across reloads, and saved in
@@ -306,6 +310,9 @@ func ParseTable(data []byte) (*Table, error) {
continue
}
mark, _, _ := strings.Cut(strings.TrimPrefix(r.Comment, "hc:"), " ")
if strings.HasPrefix(mark, render.FallbackMark+"-") {
t.Fallback = true
}
for _, e := range r.Expr {
var c struct {
Counter *Counter `json:"counter"`
@@ -411,6 +418,35 @@ type Firewall struct {
// restore is the quota usage saved before housecarld (or the machine)
// stopped, put back by the next apply.
restore *QuotaUsage
// flowDevs are the flowtable's devices in the ruleset in force, by
// name, with their interface index then (0: left out, missing).
flowDevs map[string]int
}
// ifIndex is an interface's index, 0 if it doesn't exist (tests change it).
var ifIndex = func(name string) int {
if i, err := net.InterfaceByName(name); err == nil {
return i.Index
}
return 0
}
// FlowtableStale reports whether the flowtable in force no longer matches
// the machine: a device of st's flowtable appeared, went, or came back as
// a new interface (ppp0 after PPPoE reconnects; the kernel drops a
// device from the flowtable when it goes). Applying again fixes it.
func (f *Firewall) FlowtableStale(st *model.State) bool {
f.mu.Lock()
defer f.mu.Unlock()
if f.flowDevs == nil {
return false
}
for _, d := range st.Settings.FlowtableDevices {
if ifIndex(d) != f.flowDevs[d] {
return true
}
}
return false
}
// RestoreQuotas has the next apply put this saved quota usage back where
@@ -482,6 +518,12 @@ func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
return err
}
f.restore = nil
f.flowDevs = map[string]int{}
for _, d := range st.Settings.FlowtableDevices {
if opt.Interfaces == nil || opt.Interfaces[d] {
f.flowDevs[d] = ifIndex(d)
}
}
if f.base == nil {
f.base, f.since = map[string]Counter{}, time.Now()
}
@@ -496,20 +538,37 @@ func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
return nil
}
// SaveBoot writes the ruleset of st for the boot unit, atomically.
// SaveBoot writes the ruleset of st for the boot unit, and the fallback
// it loads if that fails (FallbackFile), atomically. The boot ruleset has
// no flowtable: it loads before the network comes up, when its devices
// (ppp0, wg0) don't exist yet and nftables would refuse the whole file;
// housecarld adds it once they do.
func (f *Firewall) SaveBoot(st *model.State) error {
if f.BootFile == "" {
return nil
}
rs := render.Render(st, f.options())
tmp := f.BootFile + ".tmp"
if err := os.MkdirAll(filepath.Dir(f.BootFile), 0o750); err != nil {
return err
}
if err := os.WriteFile(tmp, []byte(rs.Text), 0o640); err != nil {
if err := writeAtomic(FallbackFile(f.BootFile), render.Fallback(st)); err != nil {
return err
}
return os.Rename(tmp, f.BootFile)
rs := render.Render(st, render.Options{Interfaces: map[string]bool{}})
return writeAtomic(f.BootFile, rs.Text)
}
// FallbackFile is where the fallback for a boot ruleset is saved:
// ruleset.nft's is ruleset-fallback.nft beside it.
func FallbackFile(boot string) string {
return strings.TrimSuffix(boot, ".nft") + "-fallback.nft"
}
func writeAtomic(path, text string) error {
tmp := path + ".tmp"
if err := os.WriteFile(tmp, []byte(text), 0o640); err != nil {
return err
}
return os.Rename(tmp, path)
}
// Table reads Housecarl's table back, with counters totalled since
+108
View File
@@ -2,8 +2,10 @@ package nft
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
@@ -257,3 +259,109 @@ func TestDeviceCountsInContainer(t *testing.T) {
t.Errorf("after a reload: %+v", e)
}
}
// TestFlowtableStale: the flowtable in force goes stale when one of its
// devices appears, goes, or comes back as a new interface.
func TestFlowtableStale(t *testing.T) {
idx := map[string]int{"eth1": 3}
old := ifIndex
ifIndex = func(name string) int { return idx[name] }
t.Cleanup(func() { ifIndex = old })
exists := func() map[string]bool {
out := map[string]bool{}
for n, i := range idx {
out[n] = i > 0
}
return out
}
// "true" stands in for nft: every apply succeeds, and lists nothing.
fw := &Firewall{Runner: &Runner{Command: []string{"true"}}, Interfaces: exists}
st := model.NewState()
st.Settings.FlowtableDevices = []string{"ppp0", "eth1"}
if fw.FlowtableStale(st) {
t.Error("stale before anything was applied")
}
if err := fw.Apply(context.Background(), st); err != nil {
t.Fatal(err)
}
if fw.FlowtableStale(st) {
t.Error("stale right after applying")
}
idx["ppp0"] = 7 // PPPoE came up
if !fw.FlowtableStale(st) {
t.Error("ppp0 appeared, yet not stale")
}
fw.Apply(context.Background(), st)
idx["ppp0"] = 9 // and reconnected: a new interface by the same name
if !fw.FlowtableStale(st) {
t.Error("ppp0 came back as a new interface, yet not stale")
}
fw.Apply(context.Background(), st)
idx["eth9"] = 4 // not a flowtable device
if fw.FlowtableStale(st) {
t.Error("stale over an interface the flowtable doesn't use")
}
}
// TestBootInContainer: the boot ruleset loads although its flowtable
// devices don't exist yet, the fallback loads and is recognised, and
// housecarld's apply replaces it.
func TestBootInContainer(t *testing.T) {
image := os.Getenv("HOUSECARL_NFT_IMAGE")
if image == "" {
t.Skip("HOUSECARL_NFT_IMAGE not set")
}
name := "housecarl-nft-boot-test"
exec.Command("docker", "rm", "-f", name).Run()
if out, err := exec.Command("docker", "run", "-d", "--name", name, "--cap-add", "NET_ADMIN", image, "sleep", "300").CombinedOutput(); err != nil {
t.Fatalf("starting the container: %v %s", err, out)
}
t.Cleanup(func() { exec.Command("docker", "rm", "-f", name).Run() })
nft := func(script string) error {
c := exec.Command("docker", "exec", "-i", name, "nft", "-f", "/dev/stdin")
c.Stdin = strings.NewReader(script)
if out, err := c.CombinedOutput(); err != nil {
return fmt.Errorf("%v: %s", err, out)
}
return nil
}
dir := t.TempDir()
// ppp0 exists when housecarld saves, but not at boot.
fw := &Firewall{Runner: &Runner{Command: []string{"docker", "exec", "-i", name, "nft"}}, BootFile: filepath.Join(dir, "ruleset.nft"),
Interfaces: func() map[string]bool { return map[string]bool{"ppp0": true, "eth0": true} }}
st := model.NewState()
st.Zones = []*model.Zone{{ID: 1, Name: "wan", Interfaces: []string{"ppp0"}, External: true}, {ID: 2, Name: "lan", Interfaces: []string{"eth0"}}}
st.SNATs = []*model.SourceNAT{{ID: 3, Match: model.Match{OutZones: []string{"wan"}}}}
st.Settings.FlowtableDevices = []string{"ppp0", "eth0"}
if err := fw.SaveBoot(st); err != nil {
t.Fatal(err)
}
boot, _ := os.ReadFile(fw.BootFile)
if strings.Contains(string(boot), "flowtable") {
t.Errorf("the boot ruleset has a flowtable:\n%s", boot)
}
fallback, err := os.ReadFile(filepath.Join(dir, "ruleset-fallback.nft"))
if err != nil {
t.Fatal(err)
}
if err := nft(string(boot)); err != nil {
t.Fatalf("the boot ruleset doesn't load: %v", err)
}
ctx := context.Background()
if tb, err := fw.Table(ctx); err != nil || tb.Fallback {
t.Errorf("after the boot ruleset: %+v %v", tb, err)
}
if err := nft(string(fallback)); err != nil {
t.Fatalf("the fallback doesn't load: %v", err)
}
if tb, err := fw.Table(ctx); err != nil || !tb.Fallback {
t.Errorf("the fallback isn't recognised: %+v %v", tb, err)
}
fw.Interfaces = func() map[string]bool { return map[string]bool{"eth0": true} }
if err := fw.Apply(ctx, st); err != nil {
t.Fatalf("apply over the fallback: %v", err)
}
if tb, err := fw.Table(ctx); err != nil || tb.Fallback {
t.Errorf("the fallback stayed after an apply: %+v %v", tb, err)
}
}
+89
View File
@@ -0,0 +1,89 @@
package render
import (
"fmt"
"strings"
"housecarl/internal/model"
)
// FallbackMark starts the comment of every fallback rule ("hc:fallback-…"):
// how housecarld tells the fallback is in force.
const FallbackMark = "fallback"
// Fallback builds the ruleset housecarl-ruleset.service loads at boot when
// the saved one fails to: nothing new gets in from the external zones'
// interfaces, everything else is let through, and IPv4 leaving an
// external zone is masqueraded (when any source NAT is configured), so the
// inside keeps working until housecarld puts the real ruleset back. With
// no external zone it can't tell which interface faces the internet, and
// lets nothing but loopback through.
//
// It is deliberately small: no sets, flowtable, meters or quotas, and
// interfaces by name only, so it loads before they exist.
func Fallback(st *model.State) string {
var b strings.Builder
line := func(format string, args ...any) {
fmt.Fprintf(&b, format, args...)
b.WriteByte('\n')
}
// rule writes a counted rule: what it matches, then its verdict.
rule := func(name, match, verdict string) {
line("\t\t%s counter %s comment \"hc:%s-%s\"", match, verdict, FallbackMark, name)
}
ext := st.ExternalInterfaces()
line("# Housecarl's fallback: loaded at boot only if ruleset.nft fails to load,")
line("# until housecarld starts and puts the real ruleset back.")
line("table inet %s {}", Table)
line("delete table inet %s", Table)
line("table inet %s {", Table)
if len(ext) == 0 {
// Nothing says which interface faces the internet: close it all.
for _, c := range []string{"input", "forward"} {
line("\tchain %s {", c)
line("\t\ttype filter hook %s priority filter; policy drop;", c)
line("\t\tiifname \"lo\" accept")
line("\t\tcounter comment \"hc:%s-%s\"", FallbackMark, c)
line("\t}")
}
line("}")
return b.String()
}
e := ifaces(ext)
line("\tchain input {")
line("\t\ttype filter hook input priority filter; policy accept;")
line("\t\tiifname \"lo\" accept")
line("\t\tct state established,related accept")
rule("icmpv6", fmt.Sprintf("iifname %s icmpv6 type { %s, echo-request, echo-reply }", e, icmpv6Errors), "accept")
rule("icmpv6-nd", fmt.Sprintf("iifname %s icmpv6 type { %s } ip6 hoplimit 255", e, icmpv6ND), "accept")
rule("icmpv6-mld", fmt.Sprintf("iifname %s ip6 saddr fe80::/10 icmpv6 type { %s }", e, icmpv6MLD), "accept")
rule("dhcpv6-client", fmt.Sprintf("iifname %s ip6 saddr fe80::/10 udp dport 546", e), "accept")
rule("input", "iifname "+e, "drop")
line("\t}")
line("\tchain forward {")
line("\t\ttype filter hook forward priority filter; policy accept;")
if st.Settings.Builtins.MSSClamp {
line("\t\toifname %s tcp flags syn tcp option maxseg size set rt mtu", e)
}
line("\t\tct state established,related accept")
rule("forward", "iifname "+e, "drop")
line("\t}")
if masquerades(st) {
line("\tchain srcnat {")
line("\t\ttype nat hook postrouting priority srcnat; policy accept;")
rule("masquerade", "meta nfproto ipv4 oifname "+e, "masquerade")
line("\t}")
}
line("}")
return b.String()
}
// masquerades reports whether any source NAT is configured.
func masquerades(st *model.State) bool {
for _, x := range st.SNATs {
if !x.Disabled {
return true
}
}
return false
}
+75
View File
@@ -0,0 +1,75 @@
package render
import (
"context"
"os"
"os/exec"
"strings"
"testing"
"housecarl/internal/model"
)
func TestFallback(t *testing.T) {
st := router()
text := Fallback(st)
for _, want := range []string{
"delete table inet housecarl",
"type filter hook input priority filter; policy accept;",
`iifname "wan" counter drop comment "hc:fallback-input"`,
`iifname "wan" counter drop comment "hc:fallback-forward"`,
`iifname "wan" ip6 saddr fe80::/10 udp dport 546 counter accept`,
`ip6 hoplimit 255 counter accept`,
`oifname "wan" tcp flags syn tcp option maxseg size set rt mtu`,
`meta nfproto ipv4 oifname "wan" counter masquerade comment "hc:fallback-masquerade"`,
} {
if !strings.Contains(text, want) {
t.Errorf("no %q in\n%s", want, text)
}
}
// Nothing that could fail to load before the network is up.
for _, not := range []string{"flowtable", "@", "\tset ", "enp2s0", "wg0"} {
if strings.Contains(text, not) {
t.Errorf("%q in the fallback:\n%s", not, text)
}
}
// Established connections are accepted before the drops.
if strings.Index(text, "ct state established,related accept") > strings.Index(text, "hc:fallback-input") {
t.Errorf("established after the drop:\n%s", text)
}
// No source NAT: no masquerade.
st.SNATs = nil
if text := Fallback(st); strings.Contains(text, "masquerade") {
t.Errorf("masquerade without source NAT:\n%s", text)
}
// No external zone: everything but loopback is dropped.
for _, z := range st.Zones {
z.External = false
}
text = Fallback(st)
if strings.Count(text, "policy drop;") != 2 || strings.Contains(text, "policy accept") || strings.Contains(text, "established") {
t.Errorf("no external zone:\n%s", text)
}
}
func TestFallbackPassesNft(t *testing.T) {
cmd := strings.Fields(os.Getenv("HOUSECARL_NFT"))
if len(cmd) == 0 {
t.Skip("HOUSECARL_NFT not set")
}
closed := router()
for _, z := range closed.Zones {
z.External = false
}
wild := router()
wild.Zones = append(wild.Zones, &model.Zone{ID: 99, Name: "lte", Interfaces: []string{"wwan*"}, External: true})
for name, st := range map[string]*model.State{"router": router(), "closed": closed, "wildcard": wild} {
c := exec.CommandContext(context.Background(), cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...)
c.Stdin = strings.NewReader(Fallback(st))
if out, err := c.CombinedOutput(); err != nil {
t.Errorf("%s: nft -c: %v\n%s\n%s", name, err, out, Fallback(st))
}
}
}
+9 -1
View File
@@ -244,7 +244,15 @@ func TestRenderedRulesetPassesNft(t *testing.T) {
}
stats := router()
stats.Settings.StatsPerDevice = true
for name, st := range map[string]*model.State{"router": router(), "empty": model.NewState(), "stats": stats} {
// VLAN sub-interfaces: zones of one VLAN, of all of a parent's, and
// the flowtable on one.
vlan := router()
vlan.Zones = append(vlan.Zones, &model.Zone{ID: 900, Name: "iot", Interfaces: []string{"enp2s0.20"}},
&model.Zone{ID: 901, Name: "vlans", Interfaces: []string{"enp3s0.*"}})
vlan.Rules = append(vlan.Rules, &model.Rule{ID: 902, Chain: model.ChainForward, Position: 99, Action: model.ActionAccept,
Match: model.Match{InZones: []string{"iot", "vlans"}, OutZones: []string{"wan"}}})
vlan.Settings.FlowtableDevices = []string{"lo", "enp2s0.20"}
for name, st := range map[string]*model.State{"router": router(), "empty": model.NewState(), "stats": stats, "vlan": vlan} {
q, qs := quotaOptions(st)
rs := Render(st, Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}, {Addr: "2a02:8013:7f3d:30::5"}}}, Quotas: q, QuotaSets: qs,
DeviceSets: map[string][]Element{"stats_up_v4": {{Addr: "10.0.10.5", Timeout: 3000, Packets: 7, Bytes: 7000}}, "stats_down_v6": {{Addr: "2001:db8:1:2::", Packets: 1, Bytes: 100}}}})
+41
View File
@@ -381,6 +381,47 @@ func usedBy(st *model.State, kind string, o model.Object) string {
return ""
}
// renameRefs makes every object that refers to a zone, set or service by
// its old name use the new one: renaming it changes them in the same
// change, so the history shows (and undoes) them together.
func renameRefs(st *model.State, kind, old, now string) {
fix := func(list []string) {
for i, x := range list {
if strings.EqualFold(x, old) {
list[i] = now
}
}
}
match := func(m *model.Match) {
switch kind {
case model.KindZone:
fix(m.InZones)
fix(m.OutZones)
case model.KindSet:
fix(m.Sources)
fix(m.Destinations)
case model.KindService:
fix(m.Services)
}
}
for _, x := range st.Rules {
match(&x.Match)
}
for _, x := range st.SNATs {
match(&x.Match)
}
for _, x := range st.Redirects {
match(&x.Match)
}
if kind == model.KindSet {
for _, f := range st.Forwards {
fix(f.Sources)
fix(f.PublicAddresses)
fix(f.HairpinSources)
}
}
}
// moveObject reorders a rule or NAT rule (moveOp).
func (s *Server) moveObject(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
kind, err := pathKind(r)
+70
View File
@@ -0,0 +1,70 @@
package server
import (
"net/http"
"sync"
"time"
"housecarl/internal/version"
)
// fallbackInForce is what status and /healthz say while the boot fallback
// (render.Fallback) is in force.
const fallbackInForce = "the boot fallback is in force, not Housecarl's ruleset: the saved ruleset failed to load at boot (journalctl -u housecarl-ruleset), and housecarld couldn't apply it since; it lets nothing new in from external zones (with none, nothing but loopback)"
// health is GET /healthz, for monitoring (Checkmk, a load balancer). It
// needs no authentication, so it says only what a monitor needs, and reads
// the ruleset at most every few seconds however often it is asked.
type health struct {
OK bool `json:"ok"`
Version string `json:"version"`
Firewall string `json:"firewall"` // applied, unconfigured, fallback, missing
// Probation: changes waiting to be confirmed, rolled back otherwise.
Probation int `json:"probation"`
// Warnings are the codes of the machine's problems (nftables_service).
Warnings []string `json:"warnings"`
}
const healthTTL = 5 * time.Second
type healthCache struct {
mu sync.Mutex
at time.Time
out health
}
func (s *Server) healthz(w http.ResponseWriter, r *http.Request) {
c := &s.health
c.mu.Lock()
if time.Since(c.at) >= healthTTL {
c.out, c.at = s.checkHealth(r), time.Now()
}
out := c.out
c.mu.Unlock()
code := http.StatusOK
if !out.OK {
code = http.StatusServiceUnavailable
}
writeJSON(w, code, out)
}
func (s *Server) checkHealth(r *http.Request) health {
out := health{Version: version.String(), Warnings: []string{}}
t, err := s.fw.Table(r.Context())
switch {
case err != nil && s.eng.State().Empty():
out.Firewall, out.OK = "unconfigured", true
case err != nil:
out.Firewall = "missing"
case t.Fallback:
out.Firewall = "fallback"
default:
out.Firewall, out.OK = "applied", true
}
pr := s.probationStatus()
out.Probation = len(pr.Pending) + pr.Takeovers
for _, p := range s.hostProblems(r.Context(), nil) {
out.Warnings = append(out.Warnings, p.Code)
}
return out
}
+65
View File
@@ -0,0 +1,65 @@
package server
import (
"encoding/json"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"housecarl/internal/api"
)
func TestHealthz(t *testing.T) {
e := newEnv(t)
link := filepath.Join(t.TempDir(), "nftables.service")
e.srv.unitLinks = []string{link}
get := func() (int, health) {
t.Helper()
e.srv.health.at = time.Time{} // no caching between the steps
resp, err := http.Get(e.ts.URL + "/healthz")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
var h health
json.NewDecoder(resp.Body).Decode(&h)
return resp.StatusCode, h
}
// Nothing configured yet: nothing to be in force.
if code, h := get(); code != 200 || !h.OK || h.Firewall != "unconfigured" || h.Version == "" {
t.Errorf("unconfigured: %d %+v", code, h)
}
if code := e.call(e.token, "POST", "/zones", map[string]any{"name": "wan", "interfaces": []string{"wan"}, "external": true}, nil); code != 201 {
t.Fatalf("creating a zone: %d", code)
}
if code, h := get(); code != 200 || h.Firewall != "applied" || h.Probation != 1 || len(h.Warnings) != 0 {
t.Errorf("applied, on probation: %d %+v", code, h)
}
os.WriteFile(link, nil, 0o644)
if _, h := get(); strings.Join(h.Warnings, ",") != "nftables_service" {
t.Errorf("warnings %v", h.Warnings)
}
e.fw.mu.Lock()
e.fw.fallback = true
e.fw.mu.Unlock()
if code, h := get(); code != 503 || h.OK || h.Firewall != "fallback" {
t.Errorf("fallback: %d %+v", code, h)
}
var st api.Status
e.call(e.token, "GET", "/status", nil, &st)
if st.Applied || !strings.Contains(st.ApplyError, "boot fallback") {
t.Errorf("status with the fallback: applied %v, %q", st.Applied, st.ApplyError)
}
// Asked often, the ruleset is read once per few seconds.
e.fw.mu.Lock()
e.fw.fallback = false
e.fw.mu.Unlock()
resp, _ := http.Get(e.ts.URL + "/healthz")
resp.Body.Close()
if resp.StatusCode != 503 {
t.Errorf("not cached: %d", resp.StatusCode)
}
}
+3 -1
View File
@@ -37,10 +37,12 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa
for _, k := range model.Kinds {
out.Counts[k] = len(st.Objects(k))
}
if _, err := s.fw.Table(r.Context()); err != nil && st.Empty() {
if t, err := s.fw.Table(r.Context()); err != nil && st.Empty() {
out.Unconfigured = true
} else if err != nil {
out.ApplyError = err.Error()
} else if t.Fallback {
out.ApplyError = fallbackInForce
} else {
out.Applied = true
}
+11
View File
@@ -94,10 +94,21 @@ func updateOp(st *model.State, op *model.Op) (int64, error) {
return 0, err
}
setID(next, id)
if old, now := o.Label(), next.Label(); old != now && isNamed(op.Kind) {
if set, ok := o.(*model.AddressSet); ok && set.Dynamic {
return 0, conflict("set %s is dynamic: its entries, and the API tokens and programs that fill it, go by its name; make a new set instead", old)
}
renameRefs(st, op.Kind, old, now)
}
data, _ := json.Marshal(next)
return id, st.Put(op.Kind, id, data)
}
// isNamed reports whether other objects refer to a kind by name.
func isNamed(kind string) bool {
return kind == model.KindZone || kind == model.KindSet || kind == model.KindService
}
func deleteOp(st *model.State, op *model.Op) error {
o, err := findObject(st, op.Kind, op.Ref)
if err != nil {
+84
View File
@@ -0,0 +1,84 @@
package server
import (
"strings"
"testing"
"housecarl/internal/api"
)
// TestRename: renaming a zone, set or service changes what refers to it,
// in the same change, which undoes as one.
func TestRename(t *testing.T) {
e := newEnv(t)
e.srv.probationFor = 0
post := func(path string, body map[string]any) {
t.Helper()
if code := e.call(e.token, "POST", path, body, nil); code != 201 {
t.Fatalf("POST %s: %d", path, code)
}
}
post("/zones", map[string]any{"name": "wan", "interfaces": []string{"wan"}, "external": true})
post("/zones", map[string]any{"name": "dhcp", "interfaces": []string{"eth1"}})
post("/sets", map[string]any{"name": "trusted_ips", "entries": []string{"10.0.10.0/24"}})
post("/sets", map[string]any{"name": "guests", "dynamic": true})
post("/services", map[string]any{"name": "jarl", "entries": []string{"tcp/8053"}})
post("/rules", map[string]any{"chain": "forward", "in_zones": []string{"dhcp"}, "out_zones": []string{"wan"}, "sources": []string{"trusted_ips"}})
post("/rules", map[string]any{"chain": "input", "in_zones": []string{"dhcp"}, "services": []string{"jarl"}})
post("/snats", map[string]any{"out_zones": []string{"wan"}, "sources": []string{"trusted_ips"}})
post("/forwards", map[string]any{"name": "jf", "ports": "8096", "to": "10.0.10.57", "public_addresses": []string{"217.155.8.194"}, "sources": []string{"trusted_ips"}})
var res api.MutationResponse
if code := e.call(e.token, "PATCH", "/zones/dhcp", map[string]any{"name": "lan"}, &res); code != 200 {
t.Fatalf("renaming the zone: %d %+v", code, res)
}
zoneRename := res.HistoryID
e.call(e.token, "PATCH", "/sets/trusted_ips", map[string]any{"name": "trusted"}, &res)
setRename := res.HistoryID
e.call(e.token, "PATCH", "/services/jarl", map[string]any{"name": "jarl-api"}, &res)
serviceRename := res.HistoryID
st := e.eng.State()
if st.Zone("dhcp") != nil || st.Zone("lan") == nil {
t.Fatalf("zones %+v", st.Zones)
}
for _, r := range st.Rules {
if strings.Join(r.InZones, ",") != "lan" {
t.Errorf("rule %d in zones %v", r.ID, r.InZones)
}
if r.Chain == "forward" && strings.Join(r.Sources, ",") != "trusted" {
t.Errorf("rule %d sources %v", r.ID, r.Sources)
}
if r.Chain == "input" && strings.Join(r.Services, ",") != "jarl-api" {
t.Errorf("rule %d services %v", r.ID, r.Services)
}
}
if s := st.SNATs[0].Sources; strings.Join(s, ",") != "trusted" {
t.Errorf("snat sources %v", s)
}
if s := st.Forwards[0].Sources; strings.Join(s, ",") != "trusted" {
t.Errorf("forward sources %v", s)
}
if ps := e.eng.Problems(); len(ps) != 0 {
for _, p := range ps {
if p.Severity == "error" {
t.Errorf("problem after renaming: %+v", p)
}
}
}
// Undoing the renames (newest first) puts the references back too.
for _, id := range []int64{serviceRename, setRename, zoneRename} {
if code := e.call(e.token, "POST", "/history/"+itoa(id)+"/revert", nil, nil); code != 200 {
t.Fatalf("undo %d: %d", id, code)
}
}
st = e.eng.State()
for _, r := range st.Rules {
if strings.Join(r.InZones, ",") != "dhcp" || r.Chain == "forward" && r.Sources[0] != "trusted_ips" || r.Chain == "input" && r.Services[0] != "jarl" {
t.Errorf("after undo: rule %+v", r.Match)
}
}
// A dynamic set keeps its name: tokens and programs go by it.
if code := e.call(e.token, "PATCH", "/sets/guests", map[string]any{"name": "visitors"}, nil); code != 409 {
t.Errorf("renaming a dynamic set: %d", code)
}
}
+1 -4
View File
@@ -10,15 +10,12 @@ import (
"time"
"housecarl/internal/api"
"housecarl/internal/version"
)
func (s *Server) routes() {
m := s.mux
p := api.Prefix
m.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": version.String()})
})
m.HandleFunc("GET /healthz", s.healthz)
// Authentication, users and tokens
m.HandleFunc("POST "+p+"/auth/login", s.public(s.login))
m.HandleFunc("POST "+p+"/auth/logout", s.authed(s.logout))
+1
View File
@@ -90,6 +90,7 @@ type Server struct {
srvs []*http.Server
probation probation
health healthCache
logs *logRing
// unitLinks are where an enabled nftables.service shows (tests change
// them).
+2 -1
View File
@@ -40,6 +40,7 @@ type fakeFW struct {
resets []string
traced string
traceLines []string
fallback bool // the boot fallback is in force
}
func (f *fakeFW) Check(_ context.Context, st *model.State) error { return nil }
@@ -64,7 +65,7 @@ func (f *fakeFW) Table(context.Context) (*nft.Table, error) {
if len(f.applied) == 0 {
return nil, errors.New("No such file or directory: list table inet housecarl")
}
return &nft.Table{Counters: map[string]nft.Counter{"r1": {Packets: 3, Bytes: 180}}, Elements: f.elements}, nil
return &nft.Table{Counters: map[string]nft.Counter{"r1": {Packets: 3, Bytes: 180}}, Elements: f.elements, Fallback: f.fallback}, nil
}
func (f *fakeFW) AddElement(_ context.Context, set, addr string, timeout int) error {
f.mu.Lock()
+2 -2
View File
@@ -223,8 +223,8 @@ func (c *checker) zones() {
}
}
}
if len(c.st.Zones) > 0 && !external {
c.add(Warning, "no_external_zone", "zones", "no zone is external: antispoofing, MSS clamping and the DHCPv6 client rule need one, and port forwards without public addresses")
if !c.st.Empty() && !external {
c.add(Warning, "no_external_zone", "zones", "no zone is external: antispoofing, MSS clamping and the DHCPv6 client rule need one, and port forwards without public addresses; and if the saved ruleset fails to load at boot, the fallback can't tell which interface faces the internet, and lets nothing but loopback through")
}
}
+26
View File
@@ -165,3 +165,29 @@ func TestHostWarnings(t *testing.T) {
}
}
}
func TestNoExternalZone(t *testing.T) {
st := base()
if got := codes(Check(st, nil)); strings.Contains(got, "no_external_zone") {
t.Errorf("an external zone, yet: [%s]", got)
}
st.Zones[0].External = false
if got := codes(Check(st, nil)); !strings.Contains(got, "no_external_zone") {
t.Errorf("no external zone, and no warning: [%s]", got)
}
// An empty configuration applies nothing, so has no fallback either.
if got := codes(Check(&model.State{Settings: model.DefaultSettings()}, nil)); strings.Contains(got, "no_external_zone") {
t.Errorf("empty configuration: [%s]", got)
}
}
func TestVLANInterfaces(t *testing.T) {
st := base()
st.Zones[1].Interfaces = []string{"enp2s0.10", "enp2s0.4094", "br0.20", "enp3s0.*"}
st.Settings.FlowtableDevices = []string{"enp2s0.10"}
for _, p := range Check(st, nil) {
if p.Severity == Error {
t.Errorf("VLAN interfaces refused: %+v", p)
}
}
}
+1 -1
View File
@@ -55,7 +55,7 @@ export async function render(el) {
field("Your prefixes", own, "Your public prefixes (your delegated IPv6 /48): they can't come from outside."),
field("Antispoof IPv4", v4),
field("Antispoof IPv6", v6),
field("Flowtable", flow, "Forwarded TCP and UDP between these devices skip the rules once established (fast path). Exact names; none: off."),
field("Flowtable", flow, "Forwarded TCP and UDP between these devices skip the rules once established (fast path): for a CPU that can't keep up with the line. Forward quotas, byte rates and traffic per device then only count first packets (docs/flowtable.md). Exact names, both sides (wan and lan); none: off."),
field("Probation", probation, "How long a change made over the network has to be confirmed before it is rolled back."),
h("label"), manual,
field("Statistics", perDevice)),