Sets from Jarl: sets holding the addresses of names in Jarl

A set can take its addresses from Jarl: every A and AAAA record of its
names (NAME@VIEW), CNAMEs followed within Jarl's zones in the same view.
housecarld reads them every jarl.refresh and after each change, keeps
only the addresses within the zones and networks the set allows, and
keeps what it last read (saved in the database and the boot ruleset)
while Jarl can't be reached. A change of the addresses applies the
ruleset again without a history entry and is audited (set.jarl).

hc set add --jarl --within, hc set jarl; the kind "from Jarl" and
Addresses… on the Sets page; GET /sets/{name}/jarl and /jarl/refresh;
validation codes set_jarl_*, status warnings jarl_*; docs/jarl.md.
e2e phase with scripts/fakejarl.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/e2e.sh

@@ -20,7 +20,8 @@ IMAGE=${E2E_IMAGE:-housecarl-nft}
 WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
 docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
 
-(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
+(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc &&
+	CGO_ENABLED=0 go build -o "$WORK/fakejarl" ./scripts/fakejarl) || exit 1
 # The image: Debian 13 with nftables and network tools. 'docker build'
 # can't reach the network on some hosts, so it is made with run + commit.
 if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
@@ -104,6 +105,7 @@ ruleset: /w/data/ruleset.nft
 backup: {dir: /w/backups}
 log: {level: debug}
 stats: {database: /w/data/stats.db}
+jarl: {server: "http://127.0.0.1:8053", token: e2e-jarl-token, refresh: 10s}
 EOF
 { cat "$WORK/housecarl.yaml"; echo "shadow: {enabled: true}"; } > "$WORK/housecarl-shadow.yaml"
 CONF=housecarl.yaml
@@ -579,6 +581,77 @@ not_contains "dhcp "
 run "the ruleset is unchanged by it" 0 R nft list chain inet housecarl input
 contains 'iifname "enp2s0" udp dport 67'
 
+# ---- a set from Jarl: names' addresses, kept up to date
+
+# A fake Jarl on the router: example.com in views internal and external.
+# jf's AAAA is jellyfin's; ssh.example.com is a CNAME for it; the internet
+# host's address is outside the LAN.
+jarl_data() { # JF_AAAA
+	cat > "$WORK/jarl.json" <<EOJ
+{"zones": [{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"},
+           {"id": 2, "name": "example.com.", "view": "external", "type": "primary"}],
+ "records": {"1": [{"name": "jf.example.com.", "type": "A", "data": "$JF4"},
+                   {"name": "jf.example.com.", "type": "AAAA", "data": "$1"},
+                   {"name": "ssh.example.com.", "type": "CNAME", "data": "jf.example.com."},
+                   {"name": "net.example.com.", "type": "AAAA", "data": "$NET6"}],
+             "2": [{"name": "ssh.example.com.", "type": "AAAA", "data": "$NET6"}]}}
+EOJ
+}
+jarl_data $JF6
+docker exec -d ${P}-router sh -c '/w/fakejarl -listen 127.0.0.1:8053 -data /w/jarl.json -token e2e-jarl-token >> /w/fakejarl.log 2>&1'
+sleep 1
+run "IPv6 ssh to jellyfin is dropped" 1 connect net $JF6 22
+run "a set from Jarl" 0 H set add jfssh --jarl ssh.example.com --within lan
+run "a name in two views needs its view" 0 H set jarl jfssh --refresh
+contains "in Jarl's views external, internal: name the view"
+run "name the views" 0 H set update jfssh --jarl ssh.example.com@internal --jarl net.example.com@internal
+JR=$(H -o id rule add forward --in wan --to jfssh --service ssh --first --comment "e2e jarl")
+run "the set's addresses, CNAME followed" 0 H set jarl jfssh --refresh
+contains "ssh.example.com → jf.example.com  *internal  *$JF4  *yes"
+contains "ssh.example.com → jf.example.com  *internal  *$JF6  *yes"
+contains "net.example.com  *internal  *$NET6  *no: not within lan"
+run "in the ruleset" 0 R nft list set inet housecarl jfssh_v6
+contains "$JF6"
+not_contains "$NET6"
+run "the address outside is warned of" 0 H status
+contains "jarl_outside"
+run "IPv6 ssh to jellyfin through the Jarl set" 0 connect net $JF6 22
+contains "^ssh$"
+# The record changes in Jarl: within the refresh, the set follows.
+jarl_data 2001:db8:7f3d:10::99
+sleep 12
+run "the new address, without a change in Housecarl" 0 R nft list set inet housecarl jfssh_v6
+contains "2001:db8:7f3d:10::99"
+not_contains "$JF6"
+run "ssh to the old address is dropped again" 1 connect net $JF6 22
+run "no history entry for it" 0 H history list
+contains "update set jfssh"
+not_contains "10::99"
+run "an audit record" 0 H audit --action set.jarl
+contains "added 2001:db8:7f3d:10::99"
+# Jarl goes away: the set keeps what it had, over a restart too.
+jarl_data $JF6
+H set jarl jfssh --refresh >/dev/null
+R sh -c 'kill $(pidof fakejarl)'
+run "Jarl down: the set keeps its addresses" 0 H set jarl jfssh --refresh
+contains "Reading Jarl failed"
+contains "$JF6  *yes"
+run "still through" 0 connect net $JF6 22
+contains "^ssh$"
+run "warned" 0 H status
+contains "jarl_unreachable"
+stop
+run "restart housecarld, Jarl still down" 0 start
+run "the set is back from the database" 0 R nft list set inet housecarl jfssh_v6
+contains "$JF6"
+run "the boot ruleset has its addresses" 0 R grep -q "$JF6" /w/data/ruleset.nft
+run "still through after the restart" 0 connect net $JF6 22
+contains "^ssh$"
+run "the Jarl rule goes" 0 H rule delete -y "$JR"
+run "the Jarl set goes" 0 H set delete -y jfssh
+run "no Jarl warning left" 0 H status
+not_contains "jarl_"
+
 # ---- restart and boot
 
 stop

scripts/fakejarl/main.go

@@ -0,0 +1,63 @@
+// Command fakejarl answers the part of Jarl's API that Housecarl's Jarl
+// sets read (GET /zones, GET /zones/{id}/records), from a JSON file read
+// again on each request, so the e2e test can change records:
+//
+//	{"zones": [{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"}],
+//	 "records": {"1": [{"name": "nas.example.com.", "type": "A", "data": "10.0.0.5"}]}}
+//
+// Usage: fakejarl -listen 127.0.0.1:8053 -data jarl.json -token SECRET
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+)
+
+type data struct {
+	Zones   []map[string]any            `json:"zones"`
+	Records map[string][]map[string]any `json:"records"`
+}
+
+func main() {
+	listen := flag.String("listen", "127.0.0.1:8053", "address to listen on")
+	file := flag.String("data", "jarl.json", "the zones and records")
+	token := flag.String("token", "", "the API token Housecarl must send")
+	flag.Parse()
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer "+*token {
+			w.WriteHeader(http.StatusUnauthorized)
+			w.Write([]byte(`{"error": {"code": "unauthorized", "message": "bad token"}}`))
+			return
+		}
+		b, err := os.ReadFile(*file)
+		var d data
+		if err == nil {
+			err = json.Unmarshal(b, &d)
+		}
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		p := strings.TrimPrefix(r.URL.Path, "/api/v1")
+		switch {
+		case p == "/zones":
+			json.NewEncoder(w).Encode(d.Zones)
+		case strings.HasPrefix(p, "/zones/") && strings.HasSuffix(p, "/records"):
+			id := strings.TrimSuffix(strings.TrimPrefix(p, "/zones/"), "/records")
+			recs := d.Records[id]
+			if recs == nil {
+				recs = []map[string]any{}
+			}
+			json.NewEncoder(w).Encode(recs)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(`{"error": {"code": "not_found", "message": "not found"}}`))
+		}
+	})
+	log.Fatal(http.ListenAndServe(*listen, nil))
+}