Shadow mode: Housecarl beside the firewall in force, changing nothing

shadow.enabled in housecarl.yaml loads Housecarl's ruleset so that it
blocks, rejects and translates nothing: drops and rejects count and
accept (an accept only ends Housecarl's chain), policies accept and log
what they would drop, NAT only counts, MSS clamping and the flowtable
are left out, and raw rules that do more than accept, drop or reject are
left out with a warning. Comments mark what would happen ([would drop]).
Filter chains run after the other firewall's (+10) or before (-10); NAT
chains always first. Every change is checked by nftables in both forms.
No probation; takeovers and closing connections are refused. The boot
fallback filters nothing. internal/tablewatch puts the table back after
a flush ruleset. Mode in status, healthz, whoami, the web UI, the log
and the tracer; hc ruleset --enforcing. docs/shadow.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/hc/firewall.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"os/exec"
 	"sort"
@@ -30,6 +31,9 @@ func statusCmd() *cobra.Command {
 			}
 			return output(st, func(w io.Writer) {
 				fmt.Fprintf(w, "Housecarl:\t%s, up since %s\n", st.Build.Label, st.Started.Local().Format("2006-01-02 15:04"))
+				if st.Mode == api.ModeShadow {
+					fmt.Fprintf(w, "Mode:\tSHADOW: the ruleset blocks, rejects and translates nothing; the other firewall decides (Housecarl's chains run %s it)\n", st.ShadowPosition)
+				}
 				if st.Applied {
 					fmt.Fprintln(w, "Ruleset:\tin force (table inet housecarl)")
 				} else if st.Unconfigured {
@@ -237,21 +241,28 @@ func onOff(name, v string) (bool, error) {
 // ---- the ruleset, counters, the log
 
 func rulesetCmd() *cobra.Command {
-	return &cobra.Command{
+	var enforcing bool
+	cmd := &cobra.Command{
 		Use: "ruleset", Short: "Print the nftables ruleset Housecarl builds", Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			c, err := newClient()
 			if err != nil {
 				return err
 			}
+			var q url.Values
+			if enforcing {
+				q = url.Values{"enforcing": {"1"}}
+			}
 			var text string
-			if err := c.Do(ctx(), "GET", "/ruleset", nil, nil, &text); err != nil {
+			if err := c.Do(ctx(), "GET", "/ruleset", q, nil, &text); err != nil {
 				return err
 			}
 			fmt.Print(text)
 			return nil
 		},
 	}
+	cmd.Flags().BoolVar(&enforcing, "enforcing", false, "in shadow mode: the ruleset loaded once it is off")
+	return cmd
 }
 
 func countersCmd() *cobra.Command {
@@ -346,6 +357,9 @@ func logCmd() *cobra.Command {
 					if e.TCPFlags != "" {
 						what += " " + e.TCPFlags
 					}
+					if e.Shadow && (e.Verdict == "drop" || e.Verdict == "reject") {
+						what += "  (shadow mode: would " + e.Verdict + ", let through)"
+					}
 					fmt.Printf("%s  %-28s %-6s %-6s %s -> %s  %s\n", e.Time.Local().Format("15:04:05"), strings.TrimSpace(e.Prefix), orDash(e.In), orDash(e.Out), src, dst, what)
 				}
 			}

cmd/hc/traffic.go

@@ -205,6 +205,13 @@ func printPacket(w io.Writer, p api.TracePacket) {
 	} else {
 		fmt.Fprintf(w, "    => %s by %s\n", strings.ToUpper(p.Verdict), decided)
 	}
+	if wd := p.WouldDrop; wd != nil {
+		name := wd.Label
+		if name == "" {
+			name = strings.TrimSpace(wd.Chain + " " + wd.Mark)
+		}
+		fmt.Fprintf(w, "    => shadow mode: Housecarl would have dropped it, by %s\n", name)
+	}
 }
 
 func matrixCmd() *cobra.Command {

cmd/housecarld/main.go

@@ -38,6 +38,7 @@ import (
 	"housecarl/internal/server"
 	"housecarl/internal/stats"
 	"housecarl/internal/store"
+	"housecarl/internal/tablewatch"
 	"housecarl/internal/trace"
 	"housecarl/internal/validate"
 	"housecarl/internal/version"
@@ -153,8 +154,12 @@ func serveCmd() *cobra.Command {
 			}
 			host := &hostif.Host{}
 			eng.Env = hostEnv(host)
-			fw := &nft.Firewall{Runner: &nft.Runner{Command: nft.Wrapper()}, Interfaces: host.Exists, BootFile: a.cfg.Ruleset}
+			fw := &nft.Firewall{Runner: &nft.Runner{Command: nft.Wrapper()}, Interfaces: host.Exists, BootFile: a.cfg.Ruleset,
+				Shadow: a.cfg.Shadow.Enabled, ShadowBefore: a.cfg.Shadow.Position == config.ShadowBefore}
 			eng.Firewall = fw
+			if fw.Shadow {
+				a.log.Warn("shadow mode: Housecarl's ruleset is loaded but blocks, rejects and translates nothing; the other firewall decides (docs/shadow.md)", "position", a.cfg.Shadow.Position)
+			}
 			for _, p := range eng.Problems() {
 				a.log.Warn("configuration problem: "+p.Message, "object", p.Object, "code", p.Code)
 			}
@@ -175,6 +180,7 @@ func serveCmd() *cobra.Command {
 				Listen: a.cfg.Listen, TLSCertFile: a.cfg.TLS.CertFile, TLSKeyFile: a.cfg.TLS.KeyFile, TrustedProxies: a.cfg.Proxies(),
 				SessionLifetime: time.Duration(a.cfg.Session.Lifetime), SessionIdle: time.Duration(a.cfg.Session.IdleTimeout),
 				CookieSecure: a.cfg.Session.CookieSecure, Socket: a.cfg.Socket, ACME: acme,
+				Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position,
 			}, eng, fw, host, a.log, web.FS())
 			// Put the ruleset in force: what was on probation when
 			// housecarld stopped is rolled back first. An empty
@@ -182,12 +188,15 @@ func serveCmd() *cobra.Command {
 			// close the machine before any rule lets anyone in.
 			// What the quotas used before housecarld (or the machine)
 			// stopped comes back with the next apply.
-			if v, err := a.store.GetMeta(ctx, server.QuotaUsageKey); err == nil && v != "" {
-				var u nft.QuotaUsage
-				if json.Unmarshal([]byte(v), &u) == nil {
-					fw.RestoreQuotas(&u)
+			restoreQuotas := func() {
+				if v, err := a.store.GetMeta(ctx, server.QuotaUsageKey); err == nil && v != "" {
+					var u nft.QuotaUsage
+					if json.Unmarshal([]byte(v), &u) == nil {
+						fw.RestoreQuotas(&u)
+					}
 				}
 			}
+			restoreQuotas()
 			if !srv.ResumeProbation(ctx) {
 				if eng.State().Empty() {
 					a.log.Info("the configuration is empty: no ruleset is applied until the first change (or 'hc import')")
@@ -215,6 +224,29 @@ func serveCmd() *cobra.Command {
 					a.log.Warn("not watching the interfaces: a flowtable device that appears later is only added by the next change", "err", err)
 				}
 			}()
+			// In shadow mode the other firewall's ruleset is reloaded now
+			// and then, and a ruleset file usually starts with flush
+			// ruleset, which deletes Housecarl's table too: put it back.
+			if fw.Shadow {
+				go func() {
+					err := tablewatch.Watch(ctx, func() {
+						if _, err := fw.Table(ctx); !errors.Is(err, nft.ErrNoTable) || eng.State().Empty() {
+							return
+						}
+						// The quotas' usage went with the table: from
+						// what was last saved.
+						restoreQuotas()
+						if err := eng.ApplyCurrent(ctx); err != nil {
+							a.log.Error("Housecarl's table was deleted, and putting it back failed", "err", err)
+							return
+						}
+						a.log.Info("Housecarl's table was deleted (flush ruleset?): put it back")
+					})
+					if err != nil {
+						a.log.Warn("not watching nftables: if Housecarl's table is deleted, only the next change or a restart puts it back", "err", err)
+					}
+				}()
+			}
 			go nflog.Listen(ctx, render.LogGroup, a.log, srv.LogPacket)
 			go srv.RunQuotas(ctx)
 			srv.SetConntrack(conntrack.Client{})