Shadow mode: Housecarl beside the firewall in force, changing nothing

shadow.enabled in housecarl.yaml loads Housecarl's ruleset so that it
blocks, rejects and translates nothing: drops and rejects count and
accept (an accept only ends Housecarl's chain), policies accept and log
what they would drop, NAT only counts, MSS clamping and the flowtable
are left out, and raw rules that do more than accept, drop or reject are
left out with a warning. Comments mark what would happen ([would drop]).
Filter chains run after the other firewall's (+10) or before (-10); NAT
chains always first. Every change is checked by nftables in both forms.
No probation; takeovers and closing connections are refused. The boot
fallback filters nothing. internal/tablewatch puts the table back after
a flush ruleset. Mode in status, healthz, whoami, the web UI, the log
and the tracer; hc ruleset --enforcing. docs/shadow.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/nft/nft.go

@@ -409,6 +409,10 @@ type Firewall struct {
 	// BootFile is where the confirmed ruleset is saved for the boot unit
 	// (housecarl-ruleset.service; "" for none).
 	BootFile string
+	// Shadow is shadow mode: the ruleset changes nothing, beside another
+	// firewall (render.Options; docs/shadow.md). ShadowBefore runs its
+	// filter chains before that one's.
+	Shadow, ShadowBefore bool
 
 	// Replacing the table resets its counters: base keeps the totals of
 	// the tables replaced since housecarld started.
@@ -478,7 +482,7 @@ func (f *Firewall) ResetQuota(ctx context.Context, ruleID int64, perAddress bool
 }
 
 func (f *Firewall) options() render.Options {
-	var o render.Options
+	o := render.Options{Shadow: f.Shadow, ShadowBefore: f.ShadowBefore}
 	if f.Interfaces != nil {
 		o.Interfaces = f.Interfaces()
 	}
@@ -488,9 +492,26 @@ func (f *Firewall) options() render.Options {
 // Render builds the ruleset for st as Apply would.
 func (f *Firewall) Render(st *model.State) *render.Ruleset { return render.Render(st, f.options()) }
 
-// Check implements engine.Firewall.
+// RenderEnforcing builds the ruleset for st as Apply would outside shadow
+// mode.
+func (f *Firewall) RenderEnforcing(st *model.State) *render.Ruleset {
+	o := f.options()
+	o.Shadow, o.ShadowBefore = false, false
+	return render.Render(st, o)
+}
+
+// Check implements engine.Firewall. In shadow mode it checks the
+// enforcing ruleset too, so that leaving shadow mode can't fail.
 func (f *Firewall) Check(ctx context.Context, st *model.State) error {
-	return f.Runner.Check(ctx, render.Render(st, f.options()))
+	if err := f.Runner.Check(ctx, render.Render(st, f.options())); err != nil {
+		return err
+	}
+	if f.Shadow {
+		if err := f.Runner.Check(ctx, f.RenderEnforcing(st)); err != nil {
+			return fmt.Errorf("the ruleset for when shadow mode is off: %w", err)
+		}
+	}
+	return nil
 }
 
 // Apply implements engine.Firewall.
@@ -542,7 +563,8 @@ func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
 // it loads if that fails (FallbackFile), atomically. The boot ruleset has
 // no flowtable: it loads before the network comes up, when its devices
 // (ppp0, wg0) don't exist yet and nftables would refuse the whole file;
-// housecarld adds it once they do.
+// housecarld adds it once they do. In shadow mode both are shadow mode's:
+// the fallback then filters nothing.
 func (f *Firewall) SaveBoot(st *model.State) error {
 	if f.BootFile == "" {
 		return nil
@@ -550,10 +572,14 @@ func (f *Firewall) SaveBoot(st *model.State) error {
 	if err := os.MkdirAll(filepath.Dir(f.BootFile), 0o750); err != nil {
 		return err
 	}
-	if err := writeAtomic(FallbackFile(f.BootFile), render.Fallback(st)); err != nil {
+	fallback := render.Fallback(st)
+	if f.Shadow {
+		fallback = render.ShadowFallback()
+	}
+	if err := writeAtomic(FallbackFile(f.BootFile), fallback); err != nil {
 		return err
 	}
-	rs := render.Render(st, render.Options{Interfaces: map[string]bool{}})
+	rs := render.Render(st, render.Options{Interfaces: map[string]bool{}, Shadow: f.Shadow, ShadowBefore: f.ShadowBefore})
 	return writeAtomic(f.BootFile, rs.Text)
 }
 

internal/nft/nft_test.go

@@ -365,3 +365,75 @@ func TestBootInContainer(t *testing.T) {
 		t.Errorf("the fallback stayed after an apply: %+v %v", tb, err)
 	}
 }
+
+// TestShadowInContainer: in shadow mode a drop counts but doesn't drop,
+// the ruleset for leaving shadow mode is checked too, and the boot files
+// are shadow mode's.
+func TestShadowInContainer(t *testing.T) {
+	image := os.Getenv("HOUSECARL_NFT_IMAGE")
+	if image == "" {
+		t.Skip("HOUSECARL_NFT_IMAGE not set")
+	}
+	name := "housecarl-nft-shadow-test"
+	exec.Command("docker", "rm", "-f", name).Run()
+	if out, err := exec.Command("docker", "run", "-d", "--name", name, "--cap-add", "NET_ADMIN", image, "sleep", "300").CombinedOutput(); err != nil {
+		t.Fatalf("starting the container: %v %s", err, out)
+	}
+	t.Cleanup(func() { exec.Command("docker", "rm", "-f", name).Run() })
+	dir := t.TempDir()
+	fw := &Firewall{Runner: &Runner{Command: []string{"docker", "exec", "-i", name, "nft"}}, Shadow: true, BootFile: filepath.Join(dir, "ruleset.nft")}
+	ctx := context.Background()
+	st := model.NewState()
+	st.Settings.OutputPolicy = model.PolicyDrop
+	st.Settings.InputPolicy = model.PolicyAccept
+	st.Settings.Builtins.Loopback = false // it would accept the ping first
+	st.Rules = []*model.Rule{{ID: 3, Chain: model.ChainOutput, Position: 1, Action: model.ActionReject, Match: model.Match{Destinations: []string{"127.0.0.2"}}}}
+	if err := fw.Apply(ctx, st); err != nil {
+		t.Fatalf("apply: %v", err)
+	}
+	ping := func() error {
+		return exec.Command("docker", "exec", name, "ping", "-c", "2", "-W", "1", "127.0.0.2").Run()
+	}
+	if err := ping(); err != nil {
+		t.Errorf("shadow mode blocked a ping: %v", err)
+	}
+	tb, err := fw.Table(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tb.Counters["r3"].Packets == 0 {
+		t.Errorf("what would be rejected isn't counted: %+v", tb.Counters)
+	}
+	// A rule shadow mode leaves out, which nftables refuses when enforcing.
+	broken := st.Clone()
+	broken.Rules = append(broken.Rules, &model.Rule{ID: 4, Chain: model.ChainInput, Position: 1, Raw: "tcp dport 22 jump nowhere", Comment: "jump"})
+	if err := fw.Check(ctx, broken); err == nil || !strings.Contains(err.Error(), "shadow mode is off") || !strings.Contains(err.Error(), "input #4 (jump)") {
+		t.Errorf("the ruleset for leaving shadow mode wasn't checked: %v", err)
+	}
+	// Enforcing, the same rule rejects.
+	enforcing := &Firewall{Runner: fw.Runner}
+	if err := enforcing.Apply(ctx, st); err != nil {
+		t.Fatal(err)
+	}
+	if ping() == nil {
+		t.Error("enforcing didn't reject")
+	}
+	// The boot files: both load, and filter nothing.
+	if err := fw.SaveBoot(st); err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range []string{"ruleset.nft", "ruleset-fallback.nft"} {
+		b, _ := os.ReadFile(filepath.Join(dir, f))
+		c := exec.Command("docker", "exec", "-i", name, "nft", "-f", "/dev/stdin")
+		c.Stdin = strings.NewReader(string(b))
+		if out, err := c.CombinedOutput(); err != nil {
+			t.Fatalf("%s doesn't load: %v %s", f, err, out)
+		}
+		if err := ping(); err != nil {
+			t.Errorf("%s blocked a ping: %v", f, err)
+		}
+	}
+	if tb, err := fw.Table(ctx); err != nil || !tb.Fallback {
+		t.Errorf("the shadow fallback isn't recognised: %+v %v", tb, err)
+	}
+}