Sets from Jarl: sets holding the addresses of names in Jarl

A set can take its addresses from Jarl: every A and AAAA record of its
names (NAME@VIEW), CNAMEs followed within Jarl's zones in the same view.
housecarld reads them every jarl.refresh and after each change, keeps
only the addresses within the zones and networks the set allows, and
keeps what it last read (saved in the database and the boot ruleset)
while Jarl can't be reached. A change of the addresses applies the
ruleset again without a history entry and is audited (set.jarl).

hc set add --jarl --within, hc set jarl; the kind "from Jarl" and
Addresses… on the Sets page; GET /sets/{name}/jarl and /jarl/refresh;
validation codes set_jarl_*, status warnings jarl_*; docs/jarl.md.
e2e phase with scripts/fakejarl.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/h_bans.go

@@ -394,7 +394,7 @@ func (s *Server) inNever(ctx context.Context, st *model.State, ref string, prefi
 		return ""
 	}
 	entries := set.Entries
-	if set.Dynamic {
+	if set.Runtime() {
 		if t, err := s.fw.Table(ctx); err == nil {
 			for _, e := range t.Elements[set.Name] {
 				entries = append(entries, e.Addr)

internal/server/h_config.go

@@ -668,6 +668,10 @@ func matrixHits(m *api.Matrix, counters map[string]nft.Counter) {
 
 // connected maps the networks on the firewall's interfaces (link-local
 // and loopback aside) to their zones.
+// Connected are the networks connected to the firewall's interfaces,
+// with their zones (link-local and loopback left out).
+func (s *Server) Connected(st *model.State) map[netip.Prefix]string { return s.connected(st) }
+
 func (s *Server) connected(st *model.State) map[netip.Prefix]string {
 	out := map[netip.Prefix]string{}
 	for _, i := range s.hostInterfaces(st) {

internal/server/h_status.go

@@ -84,16 +84,21 @@ var nftablesUnitLinks = []string{
 
 // hostProblems are warnings about the machine rather than the
 // configuration: another ruleset that still loads at boot, or the table
-// an import came from, still in force beside Housecarl's. ruleset is the
+// an import came from, still in force beside Housecarl's, and the Jarl
+// sets' names that give no addresses. ruleset is the
 // machine's (nft -j list ruleset; nil: not read). In shadow mode those
 // are expected; the warnings are that nothing else filters, and the raw
 // rules left out.
 func (s *Server) hostProblems(ctx context.Context, ruleset []byte) []validate.Problem {
+	var jarl []validate.Problem
+	if s.cfg.JarlSets != nil {
+		jarl = s.cfg.JarlSets.Problems(s.eng.State())
+	}
 	if s.cfg.Shadow {
-		return s.shadowProblems(ruleset)
+		return append(s.shadowProblems(ruleset), jarl...)
 	}
 	others := otherTables(ruleset)
-	var out []validate.Problem
+	out := jarl
 	for _, link := range s.unitLinks {
 		if _, err := os.Lstat(link); err == nil {
 			out = append(out, validate.Problem{Severity: validate.Warning, Code: "nftables_service", Object: "nftables.service",
@@ -284,6 +289,53 @@ func (s *Server) deleteElement(w http.ResponseWriter, r *http.Request, p *auth.P
 	return nil
 }
 
+// ---- Jarl sets
+
+// jarlSet is the Jarl set a request names.
+func (s *Server) jarlSet(r *http.Request, p *auth.Principal) (*model.State, *model.AddressSet, error) {
+	name := r.PathValue("ref")
+	if !p.CanSet(name, false) {
+		return nil, nil, forbidden("permission denied: this needs sets:read for %s", name)
+	}
+	st := s.eng.State()
+	set := st.Set(name)
+	if set == nil {
+		return nil, nil, notFound("no set %s", name)
+	}
+	if set.Jarl == nil {
+		return nil, nil, badRequest("set %s doesn't take its addresses from Jarl", set.Name)
+	}
+	if s.cfg.JarlSets == nil {
+		return nil, nil, conflict("this housecarld doesn't read Jarl sets")
+	}
+	return st, set, nil
+}
+
+// jarlStatus: GET /sets/{ref}/jarl.
+func (s *Server) jarlStatus(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, set, err := s.jarlSet(r, p)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, s.cfg.JarlSets.Status(st, set))
+	return nil
+}
+
+// jarlRefresh: POST /sets/{ref}/jarl/refresh reads Jarl now (every Jarl
+// set) and answers as GET does.
+func (s *Server) jarlRefresh(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if _, _, err := s.jarlSet(r, p); err != nil {
+		return err
+	}
+	s.cfg.JarlSets.Refresh(r.Context())
+	st, set, err := s.jarlSet(r, p)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, s.cfg.JarlSets.Status(st, set))
+	return nil
+}
+
 // ---- the log
 
 // logRing keeps the latest logged packets.

internal/server/jarlsets_test.go

@@ -0,0 +1,104 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"housecarl/internal/api"
+	"housecarl/internal/jarl"
+	"housecarl/internal/jarlsets"
+	"housecarl/internal/model"
+	"housecarl/internal/store"
+)
+
+// TestJarlSets: a set from Jarl, through Housecarl's Jarl client against a
+// fake Jarl, filled after a change, with its status and warnings.
+func TestJarlSets(t *testing.T) {
+	records := []map[string]string{
+		{"name": "nas.example.com.", "type": "A", "data": "10.0.10.20"},
+		{"name": "nas.example.com.", "type": "AAAA", "data": "2001:db8:10::20"},
+		{"name": "files.example.com.", "type": "CNAME", "data": "nas.example.com."},
+		{"name": "far.example.com.", "type": "A", "data": "192.0.2.9"},
+	}
+	fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer jarl-token" {
+			w.WriteHeader(401)
+			return
+		}
+		switch r.URL.Path {
+		case "/api/v1/zones":
+			json.NewEncoder(w).Encode([]map[string]any{{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"}})
+		case "/api/v1/zones/1/records":
+			if r.URL.Query().Get("disabled") != "0" {
+				t.Errorf("records asked without disabled=0: %s", r.URL)
+			}
+			json.NewEncoder(w).Encode(records)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	t.Cleanup(fake.Close)
+	jc, err := jarl.New(fake.URL, "jarl-token", "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	js := &jarlsets.Resolver{Source: jc}
+	e := newEnvWith(t, Config{JarlSets: js})
+	js.State, js.Connected = e.eng.State, e.srv.Connected
+	js.Apply = func(ctx context.Context) error { return e.eng.ApplyCurrent(ctx) }
+	js.Audit = func(ctx context.Context, set, detail string) {
+		e.eng.Audit(ctx, nil, "set.jarl", set, "success", detail)
+	}
+	ctx := context.Background()
+
+	e.call(e.token, "POST", "/zones", model.Zone{Name: "lan", Interfaces: []string{"eth1"}}, nil) // 10.0.10.0/24
+	set := model.AddressSet{Name: "nas", Jarl: &model.JarlSource{Names: []model.JarlName{{Name: "files.example.com"}, {Name: "far.example.com", View: "internal"}}, Within: []string{"lan"}}}
+	if code := e.call(e.token, "POST", "/sets", set, nil); code != 201 {
+		t.Fatalf("create: %d", code)
+	}
+	if code := e.call(e.token, "POST", "/sets", model.AddressSet{Name: "bad", Entries: []string{"10.0.0.1"}, Jarl: set.Jarl}, nil); code != 422 {
+		t.Errorf("entries in a Jarl set: %d", code)
+	}
+	var st api.JarlSet
+	if code := e.call(e.token, "POST", "/sets/nas/jarl/refresh", nil, &st); code != 200 {
+		t.Fatalf("refresh: %d", code)
+	}
+	if strings.Join(st.Entries, ",") != "10.0.10.20" || len(st.Names) != 2 || !st.Configured || st.Read == nil {
+		t.Fatalf("status %+v", st)
+	}
+	files := st.Names[0]
+	if files.Found != "internal" || strings.Join(files.Chain, ",") != "nas.example.com" || len(files.Addresses) != 2 ||
+		!files.Addresses[0].Used || files.Addresses[1].Used || !strings.Contains(files.Addresses[1].Why, "not within lan") {
+		t.Errorf("files: %+v", files)
+	}
+	var status api.Status
+	e.call(e.token, "GET", "/status", nil, &status)
+	n := 0
+	for _, p := range status.Problems {
+		if p.Code == "jarl_outside" {
+			n++
+		}
+	}
+	if n != 2 {
+		t.Errorf("jarl_outside warnings: %d (%+v)", n, status.Problems)
+	}
+	// A Jarl set can be renamed (its addresses go by its names in Jarl).
+	if code := e.call(e.token, "PATCH", "/sets/nas", map[string]any{"name": "storage"}, nil); code != 200 {
+		t.Errorf("rename: %d", code)
+	}
+	if code := e.call(e.token, "GET", "/sets/trusted/jarl", nil, nil); code != 404 {
+		t.Errorf("no such set: %d", code)
+	}
+	e.call(e.token, "POST", "/sets", model.AddressSet{Name: "trusted", Entries: []string{"10.0.10.0/24"}}, nil)
+	if code := e.call(e.token, "GET", "/sets/trusted/jarl", nil, nil); code != 400 {
+		t.Errorf("a set not from Jarl: %d", code)
+	}
+	evs, _ := e.srv.store.ListAudit(ctx, store.AuditFilter{Action: "set.jarl"})
+	if len(evs) != 1 || evs[0].Detail != "added 10.0.10.20" {
+		t.Errorf("audit %+v", evs)
+	}
+}

internal/server/routes.go

@@ -38,6 +38,8 @@ func (s *Server) routes() {
 	m.HandleFunc("GET "+p+"/sets/{ref}/elements", s.authed(s.listElements))
 	m.HandleFunc("POST "+p+"/sets/{ref}/elements", s.authed(s.addElement))
 	m.HandleFunc("DELETE "+p+"/sets/{ref}/elements/{addr}", s.authed(s.deleteElement))
+	m.HandleFunc("GET "+p+"/sets/{ref}/jarl", s.authed(s.jarlStatus))
+	m.HandleFunc("POST "+p+"/sets/{ref}/jarl/refresh", s.authed(s.jarlRefresh))
 	m.HandleFunc("GET "+p+"/bans", s.authed(s.listBans))
 	m.HandleFunc("GET "+p+"/banlists/{ref}/bans", s.authed(s.listBans))
 	m.HandleFunc("POST "+p+"/banlists/{ref}/bans", s.authed(s.addBan))

internal/server/server.go

@@ -22,6 +22,7 @@ import (
 	"housecarl/internal/auth"
 	"housecarl/internal/certs"
 	"housecarl/internal/engine"
+	"housecarl/internal/jarlsets"
 	"housecarl/internal/model"
 	"housecarl/internal/names"
 	"housecarl/internal/nft"
@@ -54,6 +55,8 @@ type Config struct {
 	ShadowPosition string
 	// Names looks up inside addresses' names (nil: names are off).
 	Names *names.Resolver
+	// JarlSets fills the sets whose addresses come from Jarl (nil: none).
+	JarlSets *jarlsets.Resolver
 }
 
 // Interfaces lists the host's interfaces with their addresses, and
@@ -153,6 +156,9 @@ func New(cfg Config, eng *engine.Engine, fw Firewall, ifaces Interfaces, log *sl
 		}
 		s.SaveQuotaUsage(context.Background())
 		s.pruneBans(context.Background(), st) // the engine is locked: not s.eng.State()
+		if cfg.JarlSets != nil {
+			cfg.JarlSets.Kick() // a new name: read it now
+		}
 	}
 	return s
 }