Release 0.2.0-rc.4

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -8,6 +8,24 @@ All notable changes to Housecarl are documented here. The format follows
 
 ### Added
 
+- **Sets from Jarl** ([docs/jarl.md](docs/jarl.md#sets-from-jarl)). A set
+  can hold the addresses of names in Jarl instead of listing them:
+  every A and AAAA record of each name, CNAMEs followed (within Jarl's
+  zones, in the same view), one view per name (`NAME@VIEW`; a name more
+  than one view covers must say which). housecarld reads them through
+  Jarl's API every `jarl.refresh` (a minute) and after each change, and
+  when they change applies the ruleset again, without a history entry
+  (audited as `set.jarl`). A set uses only the addresses `within` the
+  zones (their connected networks, which grow with the zone) and
+  networks it names; others are left out and warned of (`jarl_outside`).
+  While Jarl can't be read, sets keep the addresses they had, saved in
+  the database for restarts and in the boot ruleset (`jarl_unreachable`).
+  Names that give nothing say why (`jarl_name`: no record, a wildcard
+  only, a CNAME out of Jarl or in a loop, below a delegation, a secondary
+  zone, more than one view). `hc set add --jarl NAME@VIEW --within ZONE`,
+  `hc set jarl SET [--refresh]`; the kind "from Jarl" and Addresses… on
+  the Sets page; `GET /sets/{name}/jarl`, `POST /sets/{name}/jarl/refresh`.
+  Housecarl's Jarl token needs `records:read` for the zones.
 - An administrator can make an API token for another user in the web
   UI: New token on the user's row under Users (the CLI's `hc token create
   --user` and the API could already).
@@ -106,6 +124,8 @@ All notable changes to Housecarl are documented here. The format follows
 
 ### Changed
 
+- `hc user delete` asks first, as every other delete does, and takes
+  `-y`/`--yes` not to; it deleted the user and their tokens at once.
 - The ruleset's diff (Changes, `hc changeset diff`, `--dry-run`) names
   the chain each part is in on its `@@` line (`@@ -40,6 +40,8 @@ chain
   forward`), as `git diff` names the function, so a rule added deep in a

README.md

@@ -128,14 +128,15 @@ The web UI is on `http://127.0.0.1:8070/` (see `listen` in
 
 ## Status
 
-Housecarl 0.2.0 is in testing (0.2.0-rc.3+git62-0d16358): zones, address sets and
+Housecarl 0.2.0 is in testing (0.2.0-rc.4): zones, address sets and
 services with ordered rules, IPv4 and IPv6, port forwards with hairpin NAT
 and IPv6 pinholes, source NAT and redirects, built-ins, probation for
 network changes (confirmed by the client that made them), the ruleset
 at boot with a fail-closed fallback, hits and the drop log, nftables
 import, changesets reviewed with the ruleset's diff (as in Jarl),
 bandwidth limits and quotas, shadow mode beside another firewall, ban
-lists for Guard and fail2ban, a web UI, REST API and CLI.
+lists for Guard and fail2ban, sets of names from Jarl, a web UI, REST
+API and CLI.
 
 ## Licence
 

VERSION

@@ -1 +1 @@
-0.2.0-rc.3+git62-0d16358
+0.2.0-rc.4

cmd/hc/admin.go

@@ -349,9 +349,13 @@ operator, log) or grants such as --grant config:read --grant sets:write:guests.
 	f.BoolVar(&enable, "enable", false, "enable the user again")
 	f.StringVar(&comment, "comment", "", "a comment")
 	cmd.AddCommand(update)
-	cmd.AddCommand(&cobra.Command{
+	var yes bool
+	del := &cobra.Command{
 		Use: "delete USER", Short: "Delete a user and their tokens", Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if !confirm(fmt.Sprintf("Delete user %s and their tokens?", args[0]), yes) {
+				return fmt.Errorf("not deleted")
+			}
 			id, err := userID(args[0])
 			if err != nil {
 				return err
@@ -365,7 +369,9 @@ operator, log) or grants such as --grant config:read --grant sets:write:guests.
 			}
 			return nil
 		},
-	})
+	}
+	del.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	cmd.AddCommand(del)
 	return cmd
 }
 

cmd/hc/objects.go

@@ -271,6 +271,9 @@ func setCmd() *cobra.Command {
 				fmt.Fprintln(w, "NAME\tENTRIES\tCOMMENT")
 				for _, s := range ss {
 					entries := list(s.Entries)
+					if s.Jarl != nil {
+						entries = "from Jarl: " + strings.Join(jarlNames(s.Jarl), ", ")
+					}
 					if s.Dynamic {
 						entries = "dynamic"
 						if s.Timeout > 0 {
@@ -290,6 +293,11 @@ func setCmd() *cobra.Command {
 				fmt.Fprintf(w, "Timeout:\t%s\n", fmtSeconds(s.Timeout))
 			}
 		}
+		if s.Jarl != nil {
+			fmt.Fprintf(w, "From Jarl:\t%s\n", strings.Join(jarlNames(s.Jarl), ", "))
+			fmt.Fprintf(w, "Within:\t%s\n", strings.Join(s.Jarl.Within, ", "))
+			fmt.Fprintf(w, "Addresses:\thc set jarl %s\n", s.Name)
+		}
 		for i, e := range s.Entries {
 			label := ""
 			if i == 0 {
@@ -303,18 +311,29 @@ func setCmd() *cobra.Command {
 	}))
 	var dynamic bool
 	var timeout, comment string
+	var jarlN, within []string
 	add := &cobra.Command{
 		Use: "add NAME [ENTRY...]", Short: "Add a set", Args: cobra.MinimumNArgs(1),
 		Long: `Adds a set of addresses, prefixes and ranges, IPv4 and IPv6 mixed:
 
   hc set add trusted 192.168.1.0/24 192.168.2.0/24 2001:db8:1::/64
   hc set add guests --dynamic --timeout 24h
+  hc set add nas --jarl nas.example.com@internal --within lan
 
 A dynamic set takes entries at run time, each with its own timeout ('hc
 set entries', or the API with a token for that set alone: a captive
-portal's signed-in clients).`,
+portal's signed-in clients).
+
+A Jarl set holds the addresses of names in Jarl (their A and AAAA
+records, CNAMEs followed), read again every minute. --jarl takes
+NAME@VIEW, or NAME when only one view has it; --within says where its
+addresses may be (zones: their connected networks; or networks), and the
+others are left out ('hc set jarl NAME' shows them). docs/jarl.md.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			s := model.AddressSet{Name: args[0], Entries: args[1:], Dynamic: dynamic, Comment: comment}
+			if len(jarlN) > 0 || len(within) > 0 {
+				s.Jarl = jarlSource(jarlN, within)
+			}
 			if timeout != "" {
 				n, err := parseSeconds(timeout)
 				if err != nil {
@@ -327,6 +346,8 @@ portal's signed-in clients).`,
 	}
 	add.Flags().BoolVar(&dynamic, "dynamic", false, "entries are added at run time")
 	add.Flags().StringVar(&timeout, "timeout", "", "a dynamic set's default entry lifetime (1h, 7d)")
+	add.Flags().StringArrayVar(&jarlN, "jarl", nil, "a name in Jarl whose addresses the set holds: NAME or NAME@VIEW (repeat)")
+	add.Flags().StringArrayVar(&within, "within", nil, "a Jarl set: a zone or network its addresses must lie in (repeat)")
 	add.Flags().StringVar(&comment, "comment", "", "a comment")
 	cmd.AddCommand(add)
 	var addE, removeE, entries []string
@@ -350,6 +371,23 @@ portal's signed-in clients).`,
 				}
 				p["entries"] = editList(cur, addE, removeE)
 			}
+			if fl.Changed("jarl") || fl.Changed("within") {
+				s, err := get[model.AddressSet](objPath("sets", args[0]), nil)
+				if err != nil {
+					return err
+				}
+				j := s.Jarl
+				if j == nil {
+					j = &model.JarlSource{}
+				}
+				if fl.Changed("jarl") {
+					j.Names = jarlSource(jarlN, nil).Names
+				}
+				if fl.Changed("within") {
+					j.Within = within
+				}
+				p["jarl"] = j
+			}
 			if fl.Changed("timeout") {
 				n, err := parseSeconds(timeout)
 				if err != nil {
@@ -371,11 +409,104 @@ portal's signed-in clients).`,
 	f.StringArrayVar(&addE, "add", nil, "add an entry")
 	f.StringArrayVar(&removeE, "remove", nil, "remove an entry")
 	f.StringVar(&timeout, "timeout", "", "a dynamic set's default entry lifetime")
+	f.StringArrayVar(&jarlN, "jarl", nil, "replace a Jarl set's names: NAME or NAME@VIEW (repeat)")
+	f.StringArrayVar(&within, "within", nil, "replace where a Jarl set's addresses may be: zones, networks (repeat)")
 	f.StringVar(&comment, "comment", "", "a comment")
 	f.StringVar(&name, "name", "", "a new name")
 	cmd.AddCommand(update)
 	cmd.AddCommand(deleteCmd("sets", "set"))
 	cmd.AddCommand(entriesCmd())
+	cmd.AddCommand(jarlCmd())
+	return cmd
+}
+
+func jarlSource(names, within []string) *model.JarlSource {
+	j := &model.JarlSource{Names: []model.JarlName{}, Within: within}
+	for _, n := range names {
+		j.Names = append(j.Names, model.ParseJarlName(n))
+	}
+	return j
+}
+
+func jarlNames(j *model.JarlSource) []string {
+	var out []string
+	for _, n := range j.Names {
+		out = append(out, n.String())
+	}
+	return out
+}
+
+// jarlCmd shows what a Jarl set holds, name by name.
+func jarlCmd() *cobra.Command {
+	var refresh bool
+	cmd := &cobra.Command{Use: "jarl SET", Short: "What a Jarl set holds: each name's addresses in Jarl, and which are used", Args: cobra.ExactArgs(1),
+		Long: `Shows a Jarl set's names with the addresses Jarl gives them (CNAMEs
+followed), whether each is used or why it is left out (not within the
+set's zones and networks), and when Jarl last answered. --refresh reads
+Jarl first.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var js api.JarlSet
+			if refresh {
+				err = c.Do(ctx(), "POST", objPath("sets", args[0])+"/jarl/refresh", nil, nil, &js)
+			} else {
+				err = c.Do(ctx(), "GET", objPath("sets", args[0])+"/jarl", nil, nil, &js)
+			}
+			if err != nil {
+				return err
+			}
+			return output(js, func(w io.Writer) {
+				if !js.Configured {
+					fmt.Fprintln(w, "Jarl isn't configured (jarl.server in housecarl.yaml): the set keeps the addresses it last had.")
+				}
+				if js.Error != "" {
+					fmt.Fprintf(w, "Reading Jarl failed: %s (the set keeps the addresses it had)\n", js.Error)
+				}
+				fmt.Fprintf(w, "Set:\t%s\n", js.Set)
+				fmt.Fprintf(w, "Within:\t%s\n", strings.Join(js.Within, ", "))
+				fmt.Fprintf(w, "Holds:\t%s\n", list(js.Entries))
+				if js.Read != nil {
+					fmt.Fprintf(w, "Jarl read:\t%s\n", js.Read.Local().Format("2006-01-02 15:04:05"))
+				}
+				fmt.Fprintln(w)
+				fmt.Fprintln(w, "NAME\tVIEW\tADDRESS\tUSED")
+				for _, n := range js.Names {
+					label := n.Name
+					if len(n.Chain) > 0 {
+						label += " → " + strings.Join(n.Chain, " → ")
+					}
+					view := n.View
+					if view == "" {
+						view = n.Found
+					}
+					if view == "" {
+						view = "-"
+					}
+					switch {
+					case n.Problem != "":
+						fmt.Fprintf(w, "%s\t%s\t-\tno: %s\n", label, view, n.Problem)
+					case len(n.Addresses) == 0 && n.Error != "":
+						fmt.Fprintf(w, "%s\t%s\t-\tnot read: %s\n", label, view, n.Error)
+					case len(n.Addresses) == 0:
+						fmt.Fprintf(w, "%s\t%s\t-\tnot read yet\n", label, view)
+					}
+					for _, a := range n.Addresses {
+						used := "yes"
+						if !a.Used {
+							used = "no: " + a.Why
+						}
+						fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", label, view, a.Address, used)
+					}
+					if n.Error != "" && len(n.Addresses) > 0 {
+						fmt.Fprintf(w, "%s\t%s\t\t(the last read failed: %s; these are from before)\n", label, view, n.Error)
+					}
+				}
+			})
+		}}
+	cmd.Flags().BoolVar(&refresh, "refresh", false, "read Jarl now")
 	return cmd
 }
 

cmd/housecarld/main.go

@@ -30,6 +30,7 @@ import (
 	"housecarl/internal/engine"
 	"housecarl/internal/hostif"
 	"housecarl/internal/jarl"
+	"housecarl/internal/jarlsets"
 	"housecarl/internal/linkwatch"
 	"housecarl/internal/model"
 	"housecarl/internal/names"
@@ -165,19 +166,43 @@ func serveCmd() *cobra.Command {
 			for _, p := range eng.Problems() {
 				a.log.Warn("configuration problem: "+p.Message, "object", p.Object, "code", p.Code)
 			}
-			var acme *certs.Manager
-			if a.cfg.TLS.ACME.Domain != "" {
+			var jc *jarl.Client
+			if a.cfg.Jarl.Enabled() {
 				tok, err := a.cfg.Jarl.ReadToken()
 				if err != nil {
 					return err
 				}
-				jc, err := jarl.New(a.cfg.Jarl.Server, tok, a.cfg.Jarl.CACert, a.cfg.Jarl.Insecure)
-				if err != nil {
+				if jc, err = jarl.New(a.cfg.Jarl.Server, tok, a.cfg.Jarl.CACert, a.cfg.Jarl.Insecure); err != nil {
 					return err
 				}
+			}
+			var acme *certs.Manager
+			if a.cfg.TLS.ACME.Domain != "" {
+				if jc == nil {
+					return errors.New("tls.acme needs Jarl: set jarl.server and its token (docs/certificates.md)")
+				}
 				acme = certs.New(ctx, certs.Config{ACME: a.cfg.TLS.ACME, Version: version.Short()}, jc, a.store, a.log)
 				go acme.Run(ctx)
 			}
+			// The sets whose addresses come from Jarl: what was last read
+			// is put in force at once, Jarl is read again soon after.
+			jarlSets := &jarlsets.Resolver{Interval: time.Duration(a.cfg.Jarl.Refresh), State: eng.State, Log: a.log,
+				Apply: func(ctx context.Context) error {
+					if err := eng.ApplyCurrent(ctx); err != nil {
+						return err
+					}
+					return fw.SaveBoot(eng.State())
+				},
+				Audit: func(ctx context.Context, set, detail string) {
+					eng.Audit(ctx, &auth.Principal{Method: "jarl"}, "set.jarl", set, "success", detail)
+				},
+				Load: func(ctx context.Context) (string, error) { return a.store.GetMeta(ctx, jarlsets.MetaKey) },
+				Save: func(ctx context.Context, v string) error { return a.store.SetMeta(ctx, jarlsets.MetaKey, v) }}
+			if jc != nil {
+				jarlSets.Source = jc
+			}
+			jarlSets.Restore(ctx)
+			fw.JarlSets = jarlSets
 			var resolver *names.Resolver
 			if a.cfg.Names.Enabled {
 				resolver = names.New(a.cfg.Names.Resolver, time.Duration(a.cfg.Names.Cache))
@@ -186,8 +211,9 @@ func serveCmd() *cobra.Command {
 				Listen: a.cfg.Listen, TLSCertFile: a.cfg.TLS.CertFile, TLSKeyFile: a.cfg.TLS.KeyFile, TrustedProxies: a.cfg.Proxies(),
 				SessionLifetime: time.Duration(a.cfg.Session.Lifetime), SessionIdle: time.Duration(a.cfg.Session.IdleTimeout),
 				CookieSecure: a.cfg.Session.CookieSecure, Socket: a.cfg.Socket, ACME: acme,
-				Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position, Names: resolver,
+				Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position, Names: resolver, JarlSets: jarlSets,
 			}, eng, fw, host, a.log, web.FS())
+			jarlSets.Connected = srv.Connected
 			// Put the ruleset in force: what was on probation when
 			// housecarld stopped is rolled back first. An empty
 			// configuration applies nothing: its default policies would
@@ -217,6 +243,7 @@ func serveCmd() *cobra.Command {
 			go func() {
 				err := linkwatch.Watch(ctx, func() {
 					host.Forget()
+					jarlSets.Kick() // a zone's networks may have changed
 					if !fw.FlowtableStale(eng.State()) {
 						return
 					}
@@ -253,6 +280,7 @@ func serveCmd() *cobra.Command {
 					}
 				}()
 			}
+			go jarlSets.Run(ctx)
 			go nflog.Listen(ctx, render.LogGroup, a.log, srv.LogPacket)
 			go srv.RunQuotas(ctx)
 			srv.SetConntrack(conntrack.Client{})

docs/api.md

@@ -129,6 +129,7 @@ The objects:
 ```json
 {"name": "lan", "interfaces": ["eth1", "eth1.*"], "external": false, "comment": ""}
 {"name": "trusted", "entries": ["192.168.1.0/24", "2001:db8:1::/64"], "dynamic": false, "timeout": 0, "comment": ""}
+{"name": "nas", "entries": [], "jarl": {"names": [{"name": "nas.example.com", "view": "internal"}], "within": ["lan", "10.9.0.0/16"]}, "comment": ""}
 {"name": "web", "entries": ["tcp/8080", "tcp/8443"], "comment": ""}
 {"chain": "input",
  "in_zones": ["wan"], "not_in": false, "out_zones": [], "not_out": false,
@@ -145,6 +146,12 @@ The objects:
  "comment": "", "disabled": false}
 ```
 
+A set with `jarl` takes its addresses from Jarl ([jarl.md](jarl.md#sets-from-jarl)):
+`names` are names in Jarl, each with a `view` (left out: the only view that
+has it); `within` are zones and addresses its addresses must lie in. It
+has no `entries` and isn't `dynamic`; `"jarl": null` makes it an ordinary
+set again.
+
 A source NAT rule is a match (as a rule's, without the rule's other
 fields) with `"to"` (empty: masquerade), a redirect a match with
 `"to_port"`. A ban list ([bans.md](bans.md)):
@@ -209,6 +216,35 @@ These need `sets:read` or `sets:write`, for that set
 ([permissions.md](permissions.md)); they aren't on probation or in the
 history, but are audited.
 
+### Sets from Jarl
+
+| | |
+|---|---|
+| `GET /sets/{name}/jarl` | what the set holds and why (below) |
+| `POST /sets/{name}/jarl/refresh` | reads Jarl now (for every set from Jarl), then answers as `GET` |
+
+They need `sets:read` for that set. 400 for a set that doesn't come from
+Jarl. The answer:
+
+```json
+{"set": "nas", "configured": true, "read": "2026-10-23T14:02:11Z", "error": "",
+ "entries": ["10.10.0.20", "10.10.0.21"], "within": ["lan"],
+ "names": [{"name": "files.example.com", "view": "", "found_view": "internal", "chain": ["nas.example.com"],
+            "addresses": [{"address": "10.10.0.20", "used": true},
+                          {"address": "2001:db8:10::20", "used": false, "why": "not within lan"}],
+            "problem": "", "checked": "2026-10-23T14:02:11Z", "error": ""}]}
+```
+
+* `configured`: `housecarl.yaml` names a Jarl server; `read`: when Jarl's
+  zones were last read; `error`: why the last read failed (the set keeps
+  its addresses).
+* `entries`: the addresses the set holds now.
+* For each name: `found_view` is the view it was read from, `chain` the
+  CNAMEs followed, `addresses` every address Jarl gives it with `used`
+  (or `why` not), `problem` why it gives none, `checked` when Jarl last
+  answered for it, and `error` why the last try failed (its addresses are
+  then those of `checked`).
+
 ### Bans
 
 | | |

docs/cli.md

@@ -154,8 +154,10 @@ by name.
 | `zone delete ZONE` | refused while rules use it |
 | `set list`, `set show SET` | |
 | `set add NAME [ENTRY...] [--dynamic] [--timeout 24h] [--comment C]` | |
-| `set update SET [--entry E]... [--add E] [--remove E] [--timeout] [--name] [--comment]` | `--entry` replaces the entries |
+| `set add NAME --jarl NAME[@VIEW]... --within ZONE\|NETWORK...` | a set from Jarl: the names' addresses, only those within the zones and networks ([jarl.md](jarl.md#sets-from-jarl)) |
+| `set update SET [--entry E]... [--add E] [--remove E] [--timeout] [--jarl N]... [--within W]... [--name] [--comment]` | `--entry` replaces the entries, `--jarl` a Jarl set's names, `--within` where its addresses may be |
 | `set entries SET`, `set entries add SET ADDRESS [--timeout 1h]`, `set entries remove SET ADDRESS` | a dynamic set's entries |
+| `set jarl SET [--refresh]` | a set from Jarl: each name's addresses in Jarl (CNAMEs followed), used or why not, and when Jarl answered; `--refresh` reads Jarl first |
 | `set delete SET` | |
 | `service list`, `service show SERVICE` | with the built-in ones |
 | `service add NAME ENTRY... [--comment C]` | entries `tcp/22`, `udp/1000-2000`, `icmp/echo-request`, `icmpv6/packet-too-big` |

docs/configuration.md

@@ -13,7 +13,8 @@ reads the file when it starts.
 | `listen` | `["127.0.0.1:8070"]` | addresses of the web UI and API (`host:port`). Keep them on localhost or a management network, and write an input rule that lets you reach them |
 | `tls.cert_file`, `tls.key_file` | none | serve HTTPS with this certificate; housecarld rereads it when the file changes, or on `systemctl reload housecarl` |
 | `tls.acme.*` | off | get the certificate from Let's Encrypt, with Jarl publishing the challenge ([certificates.md](certificates.md#settings)) |
-| `jarl.server`, `jarl.token_file` (or `jarl.token`), `jarl.ca_cert`, `jarl.insecure` | none | the Jarl server for the certificate's challenge, and its token ([jarl.md](jarl.md)) |
+| `jarl.server`, `jarl.token_file` (or `jarl.token`), `jarl.ca_cert`, `jarl.insecure` | none | the Jarl server for the certificate's challenge and the sets from Jarl, and its token ([jarl.md](jarl.md)) |
+| `jarl.refresh` | `1m` | how often the sets from Jarl read their names (10s or more; [jarl.md](jarl.md#when-the-addresses-change)) |
 | `ruleset` | `/var/lib/housecarl/ruleset.nft` | the confirmed ruleset, which `housecarl-ruleset.service` loads at boot; its fallback is saved beside it as `ruleset-fallback.nft` ([operations.md](operations.md#if-the-ruleset-fails-to-load-at-boot)). Change both paths in the unit too |
 | `trusted_proxies` | `[]` | addresses or prefixes of reverse proxies whose `X-Forwarded-For` and `X-Forwarded-Proto` are believed |
 | `socket` | `/run/housecarl/housecarl.sock` | the local API socket; `""` turns it off |

docs/design.md

@@ -94,7 +94,8 @@ table inet housecarl { ... }
 Two things survive the replacement: the **entries of dynamic sets**
 (read before, written back with their remaining timeouts, in the same
 script), and the **counters** (housecarld adds the old totals to the
-new, so hits don't reset with every change).
+new, so hits don't reset with every change). Sets from Jarl are written
+with the addresses last read from Jarl.
 
 Every rule carries a comment `hc:<mark>`, such as `hc:r12` for rule 12,
 `hc:f3` for port forward 3, `hc:b-established` for a built-in. Counters
@@ -199,8 +200,15 @@ Housecarl's replaces it ([import.md](import.md)).
   may only change that set (`sets:write:guests`). Addresses, not MAC
   addresses: matching by MAC only works on the guests' own network
   segment.
-* **Names instead of addresses**: Steward knows every reservation's
-  address and Jarl every name, so rules could name a device or a subnet.
+* **Names instead of addresses**: a set can take its addresses from Jarl
+  ([jarl.md](jarl.md#sets-from-jarl)). housecarld reads the names' A and
+  AAAA records through Jarl's API (CNAMEs followed, one view per name)
+  and keeps the addresses that lie within the zones and networks the set
+  allows; Jarl never calls Housecarl. What was last read is kept (in the
+  database too) while Jarl can't be reached. A change of the addresses
+  applies the ruleset again without a history entry, as dynamic sets'
+  entries aren't in it either. Names Steward makes from leases must not
+  be used: a client chooses its own hostname.
 * **The web UI's certificate** comes through Jarl, as Steward's does
   ([certificates.md](certificates.md)).
 

docs/development.md

@@ -20,12 +20,14 @@ internal/
   server/             the HTTP API and the web UI, authentication, the local socket, probation, the log ring
   api/, client/       JSON types and the Go client shared by server and CLI
   auth/               passwords (argon2id), tokens, sessions, grants and roles
-  certs/, jarl/       the web UI's certificate through Jarl (docs/certificates.md)
+  certs/, jarl/       the web UI's certificate through Jarl (docs/certificates.md); jarl/ is the Jarl API client
+  jarlsets/           sets from Jarl: reads the names, follows CNAMEs, keeps what was read, within (docs/jarl.md)
   config/             housecarld's YAML configuration
   version/            what a build reports
   web/static/         the web UI: plain ES modules, no build step, embedded with go:embed
 deploy/               systemd units, sysusers.d and tmpfiles.d files, Debian package scripts
 scripts/e2e.sh        end-to-end test with real traffic through a router container
+scripts/fakejarl/     the bit of Jarl's API sets from Jarl read, for the e2e test
 scripts/mkdeb.sh      the Debian packages (make deb)
 scripts/logo.py       the web UI's logo from artwork/logo.svg
 scripts/banner.py     the README's banner from the web UI's logo and wordmark

docs/jarl.md

@@ -1,24 +1,44 @@
 # Connecting Housecarl to Jarl
 
 Housecarl uses [Jarl](https://gitea.adbraun.lol/adam/jarl), the BIND
-manager, for its DNS records. In this version that means one thing: the
-DNS-01 challenge for the web UI's certificate ([certificates.md](certificates.md)).
-Naming hosts in rules by their DNS names may use the same connection later.
+manager, for two things:
+
+* the DNS-01 challenge for the web UI's certificate
+  ([certificates.md](certificates.md)): Housecarl writes a TXT record;
+* **sets from Jarl** ([below](#sets-from-jarl)): sets whose addresses are
+  those of names in Jarl, such as `nas.adbraun.lol`, kept up to date.
+  Housecarl only reads.
 
 Housecarl talks to Jarl's REST API with an API token. Give it a **user of its
-own**, with **only the permission it needs**.
+own**, with **only the permission it needs**. Jarl never calls Housecarl.
 
 ## 1. A user for Housecarl, in Jarl
 
 On the Jarl server, make a user without a password. It can't log in to
-Jarl's web UI, only use its tokens. Grant it the challenge record of
-Housecarl's own name, and nothing else:
+Jarl's web UI, only use its tokens.
+
+For **sets from Jarl**, grant it reading the records of the zones and views
+your names are in:
+
+```sh
+jarl user create housecarl --no-password --comment "Housecarl on fw01" \
+    --grant "records:read zones=adbraun.lol views=internal"
+```
+
+Include every zone a name's CNAMEs lead to as well (`zones=adbraun.lol,wg.adbraun.lol`).
+Reading is all it needs: it can't change anything in Jarl.
+
+For the **certificate**, grant it the challenge record of Housecarl's own
+name, and nothing else:
 
 ```sh
 jarl user create housecarl --no-password --comment "Housecarl on fw01" \
     --grant "records:write zones=adbraun.lol names=_acme-challenge.fw.adbraun.lol types=TXT"
 ```
 
+For both, give both grants (`--grant` twice). `jarl user update housecarl
+--grant ... --grant ...` replaces a user's grants with the ones given.
+
 Use your zone for `adbraun.lol`, and the name Housecarl's web UI is reached
 by for `fw.adbraun.lol` (`tls.acme.domain`). The grant lets the token add
 and remove TXT records at exactly `_acme-challenge.fw.adbraun.lol`:
@@ -63,7 +83,9 @@ around more readily.
 You can give the token an expiry (`--expires 365d`). Rotate it in Jarl with
 `jarl token rotate ID --grace 1d`, then put the new secret in the file and
 restart housecarld within the grace period. Since the certificate is renewed
-every couple of months, nothing breaks while you do.
+every couple of months, nothing breaks while you do. Sets from Jarl keep
+their addresses while the token doesn't work (they warn, see
+[below](#when-jarl-cant-be-reached)), so they don't break either.
 
 ## 3. Housecarl's settings
 
@@ -74,6 +96,7 @@ jarl:
   server: https://dns.adbraun.lol:8053
   token_file: /etc/housecarl/jarl.token
   # ca_cert: /etc/housecarl/jarl-ca.pem   # if Jarl's certificate isn't from a public CA
+  # refresh: 1m                           # how often sets from Jarl read their names
 ```
 
 Then restart housecarld (`systemctl restart housecarl`). Housecarl needs to
@@ -83,3 +106,192 @@ address Housecarl can reach, ideally over HTTPS.
 `hc cert check` ([certificates.md](certificates.md#checking)) tests
 the connection, the token's permission, and that the record reaches the
 internet.
+
+## Sets from Jarl
+
+A set can take its addresses from Jarl instead of listing them: you name
+DNS names, and the set holds their addresses. Rules use it like any other
+set. When a record changes in Jarl, the set follows within a minute,
+without a change to Housecarl's configuration.
+
+```sh
+hc set add nas --jarl nas.adbraun.lol@internal --within lan
+hc rule add forward --in wg --to nas --service smb
+hc set jarl nas                       # what it holds, name by name
+```
+
+In the web UI: Sets → Add set, kind "from Jarl". The **Addresses…**
+button shows what the set holds.
+
+A set from Jarl has:
+
+* **names**: one or more names, each `NAME@VIEW`, or `NAME` when only one
+  of Jarl's views has it ([Views](#views));
+* **within**: where its addresses may be, one or more zones or networks
+  ([Within](#within)). It is required.
+
+It has no entries of its own, and isn't dynamic. It can be renamed (it goes
+by its names in Jarl, not by its own).
+
+### What a name gives
+
+Housecarl reads the name's records as named would answer them in the
+view:
+
+* **Every A and AAAA record** of the name. A name with three addresses
+  puts all three in the set, IPv4 and IPv6 alike; a rule using the set
+  matches each family with its own addresses.
+* **CNAMEs are followed**: `files.adbraun.lol CNAME nas.adbraun.lol.` gives
+  nas's addresses. Chains are followed up to 8 CNAMEs, from zone to zone
+  (a sub-zone such as `wg.adbraun.lol` too), but always in the **same
+  view**, and only into zones Jarl holds. `hc set jarl` shows the chain:
+  `files.adbraun.lol → nas.adbraun.lol`.
+* **Disabled records** are left out, as named leaves them out.
+
+A name gives **no addresses**, and a warning (`jarl_name`) says why, when:
+
+* it has no A, AAAA or CNAME record (only an MX, say, or nothing);
+* only a **wildcard** (`*.adbraun.lol`) would answer for it: wildcards
+  aren't followed, so name a record of its own;
+* a **CNAME leads out of Jarl** (`host.duckdns.org`), into a zone Jarl
+  doesn't hold, or to a name with no address;
+* the CNAMEs **loop**, or are more than 8;
+* it is **below a delegation**: NS records in the zone hand that part to
+  other servers, so Jarl doesn't hold its records;
+* its zone is a **secondary, mirror, stub or forward zone** (Jarl doesn't
+  hold their records), or **disabled** in Jarl;
+* no zone the token can see holds it.
+
+A **reverse zone** isn't involved: Housecarl reads the A and AAAA records
+only, never PTRs.
+
+An **in-view zone** (one view serving another's zone) is followed to the
+zone it serves: `srv.adbraun.lol@vps` reads the internal view's records if
+vps's `adbraun.lol` is an in-view of internal's.
+
+### Views
+
+The same name can have different addresses in different views: Jarl's
+`internal` view may give `nas.adbraun.lol` its private address, `external`
+a public one. A set reads **one view per name**, written `NAME@VIEW`:
+
+* **Name the view your devices see.** The firewall matches the addresses
+  packets carry, and for devices inside that is the internal view's.
+* **Without a view**, the name is read from the only view that has a zone
+  for it. If zones in **more than one view** hold it, the name gives
+  nothing until you name the view (`jarl_name`: "is in Jarl's views
+  external, internal: name the view"). This counts every view whose zones
+  cover the name, even one where the name has no record: a record added
+  there later would otherwise change which view Housecarl reads.
+* **Linked zones** make no difference: a shared record has the same
+  address in every view, and a view's own records are why the view is
+  named.
+* **Without views** in Jarl, write the name alone.
+* A set can mix views: `--jarl nas.adbraun.lol@internal --jarl
+  host.adbraun.lol@vps`.
+
+A wrong view can't widen the set: an address from it that isn't
+[within](#within) is left out.
+
+### Within
+
+`--within` says where the set's addresses may be. Each entry is:
+
+* **a zone**: the networks connected to its interfaces (the addresses on
+  them, as `hc status` lists them), IPv4 and IPv6;
+* **a network, address or range**: `10.10.0.0/24`, `2001:db8:10::/64`,
+  `192.0.2.7`.
+
+An address from Jarl that isn't within any of them is **left out** of the
+set, and a warning names it (`jarl_outside`: "nas.adbraun.lol has address
+192.0.2.9, which is left out: not within lan"). The set's other addresses
+stay. So a record changed in Jarl, by mistake or by someone who shouldn't,
+can only point the set at addresses you allowed.
+
+Zones grow by themselves: when you add a VLAN to the `lan` zone, or give
+an interface another network, its addresses count at once. A new zone
+counts only once you add it to `within`. If a zone's interfaces have no
+address for a while (at boot, a link down), the networks it had last count.
+
+Some addresses never count, whatever `within` says: unspecified (`0.0.0.0`,
+`::`), loopback and multicast ones.
+
+`within 0.0.0.0/0` or `::/0` lets any address in, and gives a warning
+(`set_jarl_anywhere`): then whoever can change the names in Jarl decides
+what the set holds. Name a zone or a network instead. For an address
+outside your networks, such as a VPS's public address, name that address
+or its network.
+
+### When the addresses change
+
+Housecarl reads the names every `jarl.refresh` (a minute), right after a
+change to Housecarl's configuration (a new set or name), when an interface
+changes, and when you ask: `hc set jarl NAME --refresh`, or **Read Jarl
+now** in the web UI.
+
+When a set's addresses change, housecarld puts the ruleset in force again
+with the new ones. That isn't a change of the configuration: there is no
+history entry and no probation. It is recorded in the audit log
+(`set.jarl`, "added 10.10.0.22; removed 10.10.0.20") and in the journal.
+
+A new set holds nothing for the moment until its names are read, usually
+less than a second.
+
+**Open connections aren't closed** when an address leaves a set, as with
+any change to a set: the connection's packets are let through as part of an
+established connection (or by the flowtable). Close them with `hc
+connections close` if they must stop at once.
+
+### When Jarl can't be reached
+
+A set **keeps the addresses it last had** while Jarl can't be read: Jarl is
+down, the network to it is, or the token stopped working. Nothing is
+emptied, so rules keep working. A warning says so (`jarl_unreachable`), on
+the status page and in `hc status`, and `hc set jarl` shows when Jarl last
+answered.
+
+What was last read is saved in the database. After a restart or a reboot,
+the sets get those addresses at once, before Jarl is read again, and the
+boot ruleset ([operations.md](operations.md)) has them too.
+
+When Jarl **answers** that a record is gone, its addresses go. Only a
+failure to read keeps them.
+
+Without `jarl.server` in `housecarl.yaml`, sets from Jarl keep what they
+had (nothing if Jarl was never read) and warn (`jarl_not_configured`).
+
+### Who decides what the set holds
+
+A set from Jarl lets **whoever can change those records in Jarl** decide
+which addresses a rule matches. Keep that in mind for rules that accept:
+
+* `within` limits it to networks you chose;
+* Housecarl's token only reads ([step 1](#1-a-user-for-housecarl-in-jarl));
+* give the records' zones only to people you'd let change the firewall,
+  and check Jarl's audit log for who changed what.
+
+**Don't use names that clients choose for themselves.** A name a DHCP
+server makes from the client's own hostname belongs to whoever asks for
+it: any guest could call itself `nas`. Name records that you made by hand,
+or that come from a DHCP reservation.
+
+### Seeing what a set holds
+
+```
+$ hc set jarl nas
+Set:        nas
+Within:     lan
+Holds:      10.10.0.20, 10.10.0.21
+Jarl read:  2026-10-23 14:02:11
+
+NAME                                 VIEW      ADDRESS          USED
+files.adbraun.lol → nas.adbraun.lol  internal  10.10.0.20       yes
+files.adbraun.lol → nas.adbraun.lol  internal  10.10.0.21       yes
+files.adbraun.lol → nas.adbraun.lol  internal  2001:db8:10::20  no: not within lan
+```
+
+The web UI's **Addresses…** button on the Sets page shows the same, and the
+status page lists the warnings: names that give nothing (`jarl_name`),
+addresses left out (`jarl_outside`), Jarl not read (`jarl_unreachable`).
+The API is `GET /sets/{name}/jarl` ([api.md](api.md#sets-from-jarl)).
+

docs/operations.md

@@ -83,7 +83,10 @@ sudo nft -c -f /var/lib/housecarl/ruleset.nft     # check it
 In shadow mode both files are shadow mode's, and the fallback filters
 nothing ([shadow.md](shadow.md#what-works-differently)).
 
-Dynamic sets start empty after a reboot. The saved ruleset has no
+Dynamic sets start empty after a reboot. Sets from Jarl have the
+addresses they last had, in the saved ruleset and again when housecarld
+starts, whether Jarl is reachable yet or not ([jarl.md](jarl.md#when-jarl-cant-be-reached)).
+The saved ruleset has no
 flowtable, since its devices may not exist yet: housecarld adds it when
 it starts, and whenever a flowtable device appears
 ([flowtable.md](flowtable.md#devices-that-come-and-go)).

docs/permissions.md

@@ -8,7 +8,7 @@ else gets **grants**, `resource:access`:
 | Resource | read | write |
 |---|---|---|
 | `config` | settings, zones, sets, services, rules, NAT, the ruleset, history, interfaces, probation, changesets (the web UI shows the forms locked) | change them, stage and apply changesets (your own, or anyone's for administrators), undo, import, confirm and roll back; close connections; start and stop traces |
-| `sets` | dynamic sets' entries | add and remove entries |
+| `sets` | dynamic sets' entries, what sets from Jarl hold | add and remove entries |
 | `bans` | ban lists' bans, and the lists the grant names | ban and unban ([bans.md](bans.md)) |
 | `log` | hit counters, the log, connections, traces | – |
 | `audit` | the audit log | – |

docs/rules.md

@@ -83,6 +83,19 @@ change to the ruleset, and a restart of housecarld; a reboot empties
 them. A token can be limited to one set ([permissions.md](permissions.md)):
 that is how another program (Steward's captive portal) lets a client in.
 
+A set **from Jarl** holds the addresses of names in Jarl, the BIND
+manager: every A and AAAA record of each name, CNAMEs followed, read again
+every minute. Only addresses within the zones and networks it names are
+used, so a changed record can't open the firewall to anything else:
+
+```sh
+hc set add nas --jarl nas.example.com@internal --within lan
+hc set jarl nas                                      # what it holds, and why
+```
+
+Views, CNAMEs, what happens when Jarl can't be reached and the rest are in
+[jarl.md](jarl.md#sets-from-jarl).
+
 ## Services
 
 A service names ports of a protocol (`tcp`, `udp`, `sctp`) and ICMP

docs/validation.md

@@ -53,9 +53,14 @@ are unique per kind, ignoring case.
 |---|---|---|
 | `set_name_reserved` | error | a set name starting with `meter_`, `conn_`, `quota_`, `stats_` or `ban_`: Housecarl's own sets (rate limits, quotas, statistics, ban lists) use those |
 | `set_entry` | error | an entry that isn't an address, prefix (without host bits) or range |
-| `set_empty` | warning | a set (not dynamic) with no entries: rules using it match nothing |
+| `set_empty` | warning | a set (not dynamic, not from Jarl) with no entries: rules using it match nothing |
 | `set_dynamic_entries` | error | configured entries on a dynamic set: its entries come at run time |
 | `set_timeout` | error | a timeout on a set that isn't dynamic, or under a second |
+| `set_jarl_entries` | error | configured entries on a set from Jarl: its addresses are its names' ([jarl.md](jarl.md#sets-from-jarl)) |
+| `set_jarl_dynamic` | error | a set both dynamic and from Jarl |
+| `set_jarl_names` | error | a set from Jarl without names, with one that isn't a DNS name (a wildcard, say) or a bad view, or with a name twice |
+| `set_jarl_within` | error | a set from Jarl without `within`, or with an entry there that is neither a zone nor an address, network or range |
+| `set_jarl_anywhere` | warning | a set from Jarl `within 0.0.0.0/0` or `::/0`: any address may come in, so whoever changes the names in Jarl decides what it holds |
 | `service_empty` | error | a service with no entries |
 | `service_entry` | error | an entry that isn't `PROTO/PORTS` (`tcp`, `udp`, `sctp`; a port or range) or `icmp/TYPE`, `icmpv6/TYPE` with a type nftables knows |
 
@@ -134,6 +139,10 @@ configuration: the dashboard and `hc status` show them.
 | `conntrack_full` | warning, *host* | the connection tracking table is over 80% full; once it is full, the kernel drops new connections. Raise `net.netfilter.nf_conntrack_max`, or find what opens so many ([stats.md](stats.md#warnings)) |
 | `takeover_pending` | warning, *host* | the table you imported from is still in force beside Housecarl's, so a packet must get past both; `hc takeover TABLE` deletes it once Housecarl's rules are right |
 | `shadow_unprotected` | warning, *host* | shadow mode, and no other table has a filter chain on the input hook: Housecarl's ruleset blocks nothing, and nothing else protects the machine. Load your firewall, or leave shadow mode ([shadow.md](shadow.md)) |
+| `jarl_name` | warning, *host* | a set from Jarl has a name that gives no addresses, and why: no A, AAAA or CNAME record, a wildcard only, a CNAME out of Jarl's zones or in a loop, below a delegation, a zone Jarl doesn't hold the records of, or in more than one view without one named ([jarl.md](jarl.md#what-a-name-gives)) |
+| `jarl_outside` | warning, *host* | an address a name has in Jarl is left out of its set: it isn't within the set's zones and networks ([jarl.md](jarl.md#within)) |
+| `jarl_unreachable` | warning, *host* | reading Jarl failed (unreachable, the token refused…), for every name or one; the sets keep the addresses they had ([jarl.md](jarl.md#when-jarl-cant-be-reached)) |
+| `jarl_not_configured` | warning, *host* | sets from Jarl, but no `jarl.server` in `housecarl.yaml`: they keep the addresses they last had |
 | `shadow_raw_left_out` | warning, *host* | shadow mode leaves this raw rule out: it does more to packets than accept, drop or reject (NAT, setting a mark or field, a jump, a verdict map…). Its hits aren't counted, and the rules after it see what it would have taken ([shadow.md](shadow.md#what-changes-in-the-ruleset)) |
 
 In shadow mode `nftables_service` and `takeover_pending` aren't raised:

internal/api/types.go

@@ -564,3 +564,43 @@ type BanMade struct {
 	model.Ban
 	Existing bool `json:"existing,omitempty"`
 }
+
+// JarlSet is what a Jarl set holds and why (GET /sets/{ref}/jarl).
+type JarlSet struct {
+	Set string `json:"set"`
+	// Configured is whether housecarl.yaml names a Jarl server.
+	Configured bool `json:"configured"`
+	// Read is when housecarld last read Jarl's zones; Error why the last
+	// try failed (the set keeps the addresses it had).
+	Read  *time.Time `json:"read,omitempty"`
+	Error string     `json:"error,omitempty"`
+	// Entries are the addresses the set holds now.
+	Entries []string   `json:"entries"`
+	Names   []JarlName `json:"names"`
+	Within  []string   `json:"within"`
+}
+
+// JarlName is one of a Jarl set's names: what Jarl gave for it.
+type JarlName struct {
+	Name string `json:"name"`
+	View string `json:"view,omitempty"` // as configured ("": the only view that has it)
+	// Found is the view it was found in.
+	Found string `json:"found_view,omitempty"`
+	// Chain are the CNAME targets followed, in order.
+	Chain     []string      `json:"chain,omitempty"`
+	Addresses []JarlAddress `json:"addresses"`
+	// Problem is why Jarl's records give it no addresses.
+	Problem string `json:"problem,omitempty"`
+	// Checked is when Jarl last answered for it; Error why the last try
+	// failed (its addresses are those of Checked).
+	Checked *time.Time `json:"checked,omitempty"`
+	Error   string     `json:"error,omitempty"`
+}
+
+// JarlAddress is an address a name has in Jarl: Used if the set holds
+// it, else why not.
+type JarlAddress struct {
+	Address string `json:"address"`
+	Used    bool   `json:"used"`
+	Why     string `json:"why,omitempty"`
+}

internal/config/config.go

@@ -153,6 +153,8 @@ type Jarl struct {
 	// CACert is a PEM file with a CA to trust for Jarl's certificate.
 	CACert   string `yaml:"ca_cert"`
 	Insecure bool   `yaml:"insecure"`
+	// Refresh is how often the Jarl sets' names are read (docs/jarl.md).
+	Refresh Duration `yaml:"refresh"`
 }
 
 // Enabled reports whether a Jarl server is configured.
@@ -238,6 +240,7 @@ func Default() *Config {
 			Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)}},
 		Shadow: Shadow{Position: ShadowAfter},
 		Names:  Names{Enabled: true, Cache: Duration(10 * time.Minute)},
+		Jarl:   Jarl{Refresh: Duration(time.Minute)},
 		TLS:    TLS{ACME: ACME{Directory: "letsencrypt-staging", Resolvers: []string{"1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"}, CheckPropagation: true}},
 	}
 }
@@ -320,6 +323,9 @@ func (c *Config) check() error {
 	if c.Names.Cache <= 0 {
 		return fmt.Errorf("names.cache must be a duration, such as 10m")
 	}
+	if c.Jarl.Refresh < Duration(10*time.Second) {
+		return fmt.Errorf("jarl.refresh must be a duration of 10s or more, such as 1m")
+	}
 	return nil
 }
 

internal/engine/engine.go

@@ -498,6 +498,15 @@ func Normalize(st *model.State) {
 	for _, x := range st.Sets {
 		x.Name = strings.TrimSpace(x.Name)
 		x.Entries = normAddrs(x.Entries)
+		if j := x.Jarl; j != nil {
+			for i, n := range j.Names {
+				j.Names[i] = model.ParseJarlName(n.Name)
+				if n.View != "" {
+					j.Names[i].View = strings.TrimSpace(n.View)
+				}
+			}
+			j.Within = normAddrs(j.Within) // zone names stay as they are
+		}
 	}
 	for _, x := range st.Services {
 		x.Name = strings.TrimSpace(x.Name)

internal/jarl/jarl.go

@@ -247,3 +247,32 @@ func (c *Client) SetTXT(ctx context.Context, name, value string, zones []Zone, s
 	}
 	return c.do(ctx, http.MethodPost, "/apply", batch{Description: desc, Ops: ops, DryRun: dryRun}, nil)
 }
+
+// ZoneInfo is a zone as Jarl lists it, of any type.
+type ZoneInfo struct {
+	ID       int64  `json:"id"`
+	Name     string `json:"name"` // "example.com."
+	View     string `json:"view,omitempty"`
+	Type     string `json:"type"`
+	InView   string `json:"in_view,omitempty"` // type in-view: the view whose zone it serves
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+// AllZones lists every zone the token can see, of any type.
+func (c *Client) AllZones(ctx context.Context) ([]ZoneInfo, error) {
+	var zs []ZoneInfo
+	return zs, c.do(ctx, http.MethodGet, "/zones", nil, &zs)
+}
+
+// Record is a record as Jarl lists it.
+type Record struct {
+	Name string `json:"name"` // "nas.example.com."
+	Type string `json:"type"`
+	Data string `json:"data"`
+}
+
+// Records lists a zone's enabled records (zone: its ID).
+func (c *Client) Records(ctx context.Context, zone int64) ([]Record, error) {
+	var rs []Record
+	return rs, c.do(ctx, http.MethodGet, fmt.Sprintf("/zones/%d/records?disabled=0", zone), nil, &rs)
+}

internal/jarlsets/jarlsets.go

@@ -0,0 +1,767 @@
+// Package jarlsets fills the sets whose addresses come from Jarl. It reads
+// the A and AAAA records of the sets' names through Jarl's API, following
+// CNAMEs within the zones Jarl holds, every minute or so, and keeps what it
+// last read (in the database too) for when Jarl can't be reached. The
+// firewall gets, for each set, the addresses that lie within what the set
+// allows (zones' connected networks, or networks it names); the others are
+// left out and reported. When what the sets hold changes, the ruleset is
+// applied again (no history entry: the configuration didn't change).
+package jarlsets
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/netip"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"housecarl/internal/api"
+	"housecarl/internal/jarl"
+	"housecarl/internal/model"
+	"housecarl/internal/validate"
+)
+
+// MetaKey is the database meta key holding what was last read.
+const MetaKey = "jarl_sets"
+
+// maxChain is how many CNAMEs are followed for one name.
+const maxChain = 8
+
+// Source is Jarl (internal/jarl.Client).
+type Source interface {
+	AllZones(ctx context.Context) ([]jarl.ZoneInfo, error)
+	Records(ctx context.Context, zone int64) ([]jarl.Record, error)
+}
+
+// Lookup is what Jarl gave for one name in one view.
+type Lookup struct {
+	Name      string   `json:"name"`
+	View      string   `json:"view,omitempty"`
+	Found     string   `json:"found,omitempty"`
+	Chain     []string `json:"chain,omitempty"`
+	Addresses []string `json:"addresses,omitempty"`
+	// Problem is why Jarl's records give no addresses (Jarl answered).
+	Problem string `json:"problem,omitempty"`
+	// Checked is when Jarl last answered; Error why the last try failed.
+	Checked time.Time `json:"checked"`
+	Error   string    `json:"error,omitempty"`
+}
+
+// saved is what the database keeps.
+type saved struct {
+	Lookups map[string]*Lookup `json:"lookups"`
+	// Nets are the zones' connected networks last seen, for when a zone's
+	// interfaces have no address (at boot, a link down).
+	Nets map[string][]netip.Prefix `json:"nets,omitempty"`
+}
+
+// Resolver keeps the Jarl sets' addresses.
+type Resolver struct {
+	// Source is Jarl (nil: not configured; the sets keep what was read).
+	Source Source
+	// Interval is how often Jarl is read (default a minute).
+	Interval time.Duration
+	// State is the configuration in force.
+	State func() *model.State
+	// Connected are the networks connected to the firewall, with their
+	// zones.
+	Connected func(st *model.State) map[netip.Prefix]string
+	// Apply puts the sets' new addresses in force.
+	Apply func(ctx context.Context) error
+	// Audit records a change of a set's addresses.
+	Audit func(ctx context.Context, set, detail string)
+	// Load and Save read and write the database's copy.
+	Load func(ctx context.Context) (string, error)
+	Save func(ctx context.Context, value string) error
+	Log  *slog.Logger
+
+	run  sync.Mutex // one refresh at a time
+	mu   sync.Mutex
+	data saved
+	// applied is what the ruleset in force holds (known after an apply).
+	applied map[string][]string
+	known   bool
+	read    time.Time
+	err     string
+	kick    chan struct{}
+	now     func() time.Time
+}
+
+func (r *Resolver) init() {
+	if r.data.Lookups == nil {
+		r.data.Lookups = map[string]*Lookup{}
+	}
+	if r.data.Nets == nil {
+		r.data.Nets = map[string][]netip.Prefix{}
+	}
+	if r.kick == nil {
+		r.kick = make(chan struct{}, 1)
+	}
+	if r.now == nil {
+		r.now = time.Now
+	}
+	if r.Log == nil {
+		r.Log = slog.Default()
+	}
+}
+
+// Restore reads what was saved before housecarld (or the machine)
+// stopped. Call it before the first apply.
+func (r *Resolver) Restore(ctx context.Context) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	if r.Load == nil {
+		return
+	}
+	v, err := r.Load(ctx)
+	if err != nil || v == "" {
+		return
+	}
+	var s saved
+	if err := json.Unmarshal([]byte(v), &s); err != nil {
+		r.Log.Warn("the saved Jarl set addresses can't be read; reading Jarl again", "err", err)
+		return
+	}
+	if s.Lookups != nil {
+		r.data.Lookups = s.Lookups
+	}
+	if s.Nets != nil {
+		r.data.Nets = s.Nets
+	}
+}
+
+// Kick has the next refresh happen now (after a change of the
+// configuration: a new name).
+func (r *Resolver) Kick() {
+	r.mu.Lock()
+	r.init()
+	k := r.kick
+	r.mu.Unlock()
+	select {
+	case k <- struct{}{}:
+	default:
+	}
+}
+
+// Run reads Jarl until ctx ends.
+func (r *Resolver) Run(ctx context.Context) {
+	r.mu.Lock()
+	r.init()
+	k := r.kick
+	every := r.Interval
+	r.mu.Unlock()
+	if every <= 0 {
+		every = time.Minute
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		r.Refresh(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		case <-k:
+		}
+	}
+}
+
+// Refresh reads Jarl now and puts what changed in force.
+func (r *Resolver) Refresh(ctx context.Context) {
+	r.run.Lock()
+	defer r.run.Unlock()
+	st := r.State()
+	r.read1(ctx, st)
+	r.sync(ctx, st)
+}
+
+// needed are the names the sets of st look up, by key.
+func needed(st *model.State) map[string]model.JarlName {
+	out := map[string]model.JarlName{}
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		for _, n := range s.Jarl.Names {
+			out[n.String()] = n
+		}
+	}
+	return out
+}
+
+// read1 reads every name from Jarl once.
+func (r *Resolver) read1(ctx context.Context, st *model.State) {
+	names := needed(st)
+	r.mu.Lock()
+	r.init()
+	src := r.Source
+	r.mu.Unlock()
+	// The zones' networks, for when they can't be seen (Entries).
+	r.rememberNets(st)
+	if src == nil || len(names) == 0 {
+		r.mu.Lock()
+		changed := false
+		for k := range r.data.Lookups {
+			if _, ok := names[k]; !ok {
+				delete(r.data.Lookups, k)
+				changed = true
+			}
+		}
+		r.mu.Unlock()
+		if changed {
+			r.save(ctx)
+		}
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	zones, err := src.AllZones(ctx)
+	now := r.now()
+	results := map[string]*Lookup{}
+	if err == nil {
+		l := &lookup{ctx: ctx, src: src, zones: zones, records: map[int64][]jarl.Record{}, errs: map[int64]error{}}
+		for k, n := range names {
+			results[k] = l.resolve(n)
+		}
+	}
+	r.mu.Lock()
+	if err != nil {
+		r.err = err.Error()
+	} else {
+		r.err, r.read = "", now
+	}
+	for k, n := range names {
+		old := r.data.Lookups[k]
+		res := results[k]
+		switch {
+		case res != nil && res.Error == "":
+			res.Checked = now
+			r.data.Lookups[k] = res
+		case old != nil:
+			// Keep what Jarl last said.
+			old.Error = errText(err, res)
+		default:
+			r.data.Lookups[k] = &Lookup{Name: n.Name, View: n.View, Error: errText(err, res)}
+		}
+		if err != nil || res != nil && res.Error != "" {
+			r.Log.Warn("reading a Jarl set's name failed; the set keeps the addresses it had", "name", n.String(), "err", r.data.Lookups[k].Error)
+		}
+	}
+	for k := range r.data.Lookups {
+		if _, ok := names[k]; !ok {
+			delete(r.data.Lookups, k)
+		}
+	}
+	r.mu.Unlock()
+	r.save(ctx)
+}
+
+func errText(err error, res *Lookup) string {
+	if err != nil {
+		return err.Error()
+	}
+	return res.Error
+}
+
+func (r *Resolver) save(ctx context.Context) {
+	if r.Save == nil {
+		return
+	}
+	r.mu.Lock()
+	b, err := json.Marshal(r.data)
+	r.mu.Unlock()
+	if err == nil {
+		err = r.Save(ctx, string(b))
+	}
+	if err != nil {
+		r.Log.Warn("saving the Jarl set addresses failed", "err", err)
+	}
+}
+
+// rememberNets keeps each zone's connected networks.
+func (r *Resolver) rememberNets(st *model.State) {
+	if r.Connected == nil {
+		return
+	}
+	now := zoneNets(r.Connected(st))
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for z, ps := range now {
+		r.data.Nets[z] = ps
+	}
+}
+
+func zoneNets(conn map[netip.Prefix]string) map[string][]netip.Prefix {
+	out := map[string][]netip.Prefix{}
+	for p, z := range conn {
+		out[z] = append(out[z], p)
+	}
+	for _, ps := range out {
+		sort.Slice(ps, func(i, j int) bool { return ps[i].String() < ps[j].String() })
+	}
+	return out
+}
+
+// sync applies the ruleset again if the sets' addresses changed.
+func (r *Resolver) sync(ctx context.Context, st *model.State) {
+	want := r.Entries(st)
+	r.mu.Lock()
+	known, applied := r.known, r.applied
+	r.mu.Unlock()
+	// No Jarl sets, nothing to do (and an empty configuration is never
+	// applied: its policies would close the machine).
+	if len(want) == 0 || st.Empty() || known && sameEntries(applied, want) || r.Apply == nil {
+		return
+	}
+	if err := r.Apply(ctx); err != nil {
+		r.Log.Error("putting the Jarl sets' new addresses in force failed", "err", err)
+		return
+	}
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		add, del := diff(applied[s.Name], want[s.Name])
+		if len(add) == 0 && len(del) == 0 {
+			continue
+		}
+		var parts []string
+		if len(add) > 0 {
+			parts = append(parts, "added "+strings.Join(add, ", "))
+		}
+		if len(del) > 0 {
+			parts = append(parts, "removed "+strings.Join(del, ", "))
+		}
+		detail := strings.Join(parts, "; ")
+		r.Log.Info("a Jarl set's addresses changed", "set", s.Name, "change", detail)
+		if r.Audit != nil {
+			r.Audit(ctx, s.Name, detail)
+		}
+	}
+}
+
+func sameEntries(a, b map[string][]string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for k, v := range a {
+		w, ok := b[k]
+		if !ok || strings.Join(v, ",") != strings.Join(w, ",") {
+			return false
+		}
+	}
+	return true
+}
+
+func diff(old, now []string) (add, del []string) {
+	was := map[string]bool{}
+	for _, a := range old {
+		was[a] = true
+	}
+	is := map[string]bool{}
+	for _, a := range now {
+		is[a] = true
+		if !was[a] {
+			add = append(add, a)
+		}
+	}
+	for _, a := range old {
+		if !is[a] {
+			del = append(del, a)
+		}
+	}
+	return add, del
+}
+
+// Applied implements nft.JarlSets.
+func (r *Resolver) Applied(entries map[string][]string) {
+	c := map[string][]string{}
+	for k, v := range entries {
+		c[k] = append([]string(nil), v...)
+	}
+	r.mu.Lock()
+	r.applied, r.known = c, true
+	r.mu.Unlock()
+}
+
+// Entries implements nft.JarlSets: every Jarl set of st (empty or not),
+// with the addresses its names have that lie within what it allows.
+func (r *Resolver) Entries(st *model.State) map[string][]string {
+	out := map[string][]string{}
+	nets := r.nets(st)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		seen := map[string]bool{}
+		list := []string{}
+		for _, n := range s.Jarl.Names {
+			lu := r.data.Lookups[n.String()]
+			if lu == nil {
+				continue
+			}
+			for _, a := range lu.Addresses {
+				if seen[a] || allowed(a, s.Jarl.Within, nets) != "" {
+					continue
+				}
+				seen[a] = true
+				list = append(list, a)
+			}
+		}
+		sortAddrs(list)
+		out[s.Name] = list
+	}
+	return out
+}
+
+// nets are the zones' connected networks: now, or as last seen.
+func (r *Resolver) nets(st *model.State) map[string][]netip.Prefix {
+	var now map[string][]netip.Prefix
+	if r.Connected != nil {
+		now = zoneNets(r.Connected(st))
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	out := map[string][]netip.Prefix{}
+	for _, z := range st.Zones {
+		if ps := now[z.Name]; len(ps) > 0 {
+			out[z.Name] = ps
+		} else {
+			out[z.Name] = r.data.Nets[z.Name]
+		}
+	}
+	return out
+}
+
+func sortAddrs(list []string) {
+	sort.Slice(list, func(i, j int) bool {
+		a, _ := netip.ParseAddr(list[i])
+		b, _ := netip.ParseAddr(list[j])
+		return a.Less(b)
+	})
+}
+
+// allowed says why an address isn't let in ("" if it is).
+func allowed(addr string, within []string, nets map[string][]netip.Prefix) string {
+	a, err := netip.ParseAddr(addr)
+	if err != nil {
+		return "not an address"
+	}
+	a = a.Unmap()
+	switch {
+	case a.IsUnspecified(), a.IsLoopback(), a.IsMulticast():
+		return "not a host's address (unspecified, loopback or multicast)"
+	}
+	for _, w := range within {
+		if l, err := model.ParseAddress(w); err == nil {
+			if l.Contains(a) {
+				return ""
+			}
+			continue
+		}
+		for _, p := range nets[w] {
+			if p.Contains(a) {
+				return ""
+			}
+		}
+	}
+	return "not within " + strings.Join(within, ", ")
+}
+
+// ---- looking names up
+
+// lookup reads Jarl's records for one refresh.
+type lookup struct {
+	ctx     context.Context
+	src     Source
+	zones   []jarl.ZoneInfo
+	records map[int64][]jarl.Record
+	errs    map[int64]error
+}
+
+func fqdn(n string) string {
+	n = strings.ToLower(strings.TrimSpace(n))
+	if !strings.HasSuffix(n, ".") {
+		n += "."
+	}
+	return n
+}
+
+func under(name, zone string) bool {
+	return zone == "." || name == zone || strings.HasSuffix(name, "."+zone)
+}
+
+func short(n string) string { return strings.TrimSuffix(n, ".") }
+
+// zoneFor finds the zone of Jarl's that answers for name in view ("":
+// the only view whose zones hold it). It returns the zone, the view, or
+// why there is none.
+func (l *lookup) zoneFor(name, view string) (*jarl.ZoneInfo, string, string) {
+	if view == "" {
+		views := map[string]bool{}
+		for _, z := range l.zones {
+			if z.Type != "hint" && under(name, fqdn(z.Name)) {
+				views[z.View] = true
+			}
+		}
+		switch len(views) {
+		case 0:
+			return nil, "", fmt.Sprintf("no zone in Jarl holds %s (or the token can't see it)", short(name))
+		case 1:
+			for v := range views {
+				view = v
+			}
+		default:
+			var vs []string
+			for v := range views {
+				vs = append(vs, v)
+			}
+			sort.Strings(vs)
+			return nil, "", fmt.Sprintf("%s is in Jarl's views %s: name the view, such as %s@%s", short(name), strings.Join(vs, ", "), short(name), vs[0])
+		}
+	}
+	var best *jarl.ZoneInfo
+	for i := range l.zones {
+		z := &l.zones[i]
+		zn := fqdn(z.Name)
+		if z.Type == "hint" || !strings.EqualFold(z.View, view) || !under(name, zn) {
+			continue
+		}
+		if best == nil || len(zn) > len(fqdn(best.Name)) {
+			best = z
+		}
+	}
+	where := ""
+	if view != "" {
+		where = " in view " + view
+	}
+	if best == nil {
+		return nil, view, fmt.Sprintf("no zone%s holds %s in Jarl (or the token can't see it)", where, short(name))
+	}
+	if best.Type == "in-view" {
+		var served *jarl.ZoneInfo
+		for i := range l.zones {
+			z := &l.zones[i]
+			if fqdn(z.Name) == fqdn(best.Name) && strings.EqualFold(z.View, best.InView) && z.Type != "in-view" {
+				served = z
+			}
+		}
+		if served == nil {
+			return nil, view, fmt.Sprintf("zone %s%s serves view %s's zone, which the token can't see", short(best.Name), where, best.InView)
+		}
+		best = served
+	}
+	if best.Disabled {
+		return nil, view, fmt.Sprintf("zone %s%s is disabled in Jarl", short(best.Name), where)
+	}
+	if best.Type != "primary" {
+		return nil, view, fmt.Sprintf("zone %s%s is a %s zone: Jarl doesn't hold its records", short(best.Name), where, best.Type)
+	}
+	return best, view, ""
+}
+
+func (l *lookup) zoneRecords(id int64) ([]jarl.Record, error) {
+	if err, ok := l.errs[id]; ok {
+		return nil, err
+	}
+	if rs, ok := l.records[id]; ok {
+		return rs, nil
+	}
+	rs, err := l.src.Records(l.ctx, id)
+	if err != nil {
+		l.errs[id] = err
+		return nil, err
+	}
+	l.records[id] = rs
+	return rs, nil
+}
+
+// resolve looks a name up as named would answer it in the view: its A
+// and AAAA records, or those of the names its CNAMEs lead to.
+func (l *lookup) resolve(n model.JarlName) *Lookup {
+	out := &Lookup{Name: n.Name, View: n.View}
+	name, view := fqdn(n.Name), n.View
+	seen := map[string]bool{name: true}
+	for hop := 0; ; hop++ {
+		z, v, problem := l.zoneFor(name, view)
+		if hop == 0 {
+			out.Found = v
+		}
+		if problem != "" {
+			if hop > 0 {
+				problem = fmt.Sprintf("its CNAME leads to %s, which Housecarl can't follow: %s (only names Jarl holds are followed)", short(name), problem)
+			}
+			out.Problem = problem
+			return out
+		}
+		view = v
+		recs, err := l.zoneRecords(z.ID)
+		if err != nil {
+			out.Error = err.Error()
+			return out
+		}
+		apex := fqdn(z.Name)
+		var addrs []string
+		cname := ""
+		for _, rec := range recs {
+			owner := fqdn(rec.Name)
+			switch {
+			case rec.Type == "NS" && owner != apex && under(name, owner):
+				out.Problem = fmt.Sprintf("%s is below a delegation (NS records at %s): Jarl doesn't hold its records", short(name), short(owner))
+				return out
+			case owner != name:
+			case rec.Type == "A" || rec.Type == "AAAA":
+				if a, err := netip.ParseAddr(strings.TrimSpace(rec.Data)); err == nil {
+					addrs = append(addrs, a.Unmap().String())
+				}
+			case rec.Type == "CNAME":
+				cname = target(rec.Data, apex)
+			}
+		}
+		if cname != "" {
+			if seen[cname] || len(out.Chain) >= maxChain {
+				out.Problem = fmt.Sprintf("its CNAMEs go round in a loop or are longer than %d (%s)", maxChain, strings.Join(append(out.Chain, short(cname)), " → "))
+				return out
+			}
+			seen[cname] = true
+			out.Chain = append(out.Chain, short(cname))
+			name = cname
+			continue
+		}
+		if len(addrs) == 0 {
+			if hop > 0 {
+				out.Problem = fmt.Sprintf("its CNAME leads to %s, which has no A or AAAA record in Jarl", short(name))
+			} else {
+				out.Problem = fmt.Sprintf("%s has no A, AAAA or CNAME record in Jarl (wildcards aren't followed)", short(name))
+			}
+			return out
+		}
+		sortAddrs(addrs)
+		out.Addresses = dedup(addrs)
+		return out
+	}
+}
+
+// target is a CNAME's target as an FQDN (a relative one is under the
+// zone, as in a zone file).
+func target(data, apex string) string {
+	t := strings.ToLower(strings.TrimSpace(data))
+	if strings.HasSuffix(t, ".") {
+		return t
+	}
+	if t == "@" {
+		return apex
+	}
+	if apex == "." {
+		return t + "."
+	}
+	return t + "." + apex
+}
+
+func dedup(in []string) []string {
+	var out []string
+	for i, a := range in {
+		if i == 0 || a != in[i-1] {
+			out = append(out, a)
+		}
+	}
+	return out
+}
+
+// ---- status
+
+// Status is what a Jarl set holds and why.
+func (r *Resolver) Status(st *model.State, set *model.AddressSet) api.JarlSet {
+	out := api.JarlSet{Set: set.Name, Entries: []string{}, Names: []api.JarlName{}, Within: []string{}}
+	if set.Jarl == nil {
+		return out
+	}
+	out.Within = append(out.Within, set.Jarl.Within...)
+	out.Entries = r.Entries(st)[set.Name]
+	nets := r.nets(st)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	out.Configured = r.Source != nil
+	if !r.read.IsZero() {
+		t := r.read
+		out.Read = &t
+	}
+	out.Error = r.err
+	for _, n := range set.Jarl.Names {
+		jn := api.JarlName{Name: n.Name, View: n.View, Addresses: []api.JarlAddress{}}
+		if lu := r.data.Lookups[n.String()]; lu != nil {
+			jn.Found, jn.Chain, jn.Problem, jn.Error = lu.Found, lu.Chain, lu.Problem, lu.Error
+			if !lu.Checked.IsZero() {
+				t := lu.Checked
+				jn.Checked = &t
+			}
+			for _, a := range lu.Addresses {
+				why := allowed(a, set.Jarl.Within, nets)
+				jn.Addresses = append(jn.Addresses, api.JarlAddress{Address: a, Used: why == "", Why: why})
+			}
+		}
+		out.Names = append(out.Names, jn)
+	}
+	return out
+}
+
+// Problems are the warnings about the Jarl sets for the status page.
+func (r *Resolver) Problems(st *model.State) []validate.Problem {
+	var out []validate.Problem
+	var sets []*model.AddressSet
+	for _, s := range st.Sets {
+		if s.Jarl != nil {
+			sets = append(sets, s)
+		}
+	}
+	if len(sets) == 0 {
+		return nil
+	}
+	r.mu.Lock()
+	configured, readErr := r.Source != nil, r.err
+	r.mu.Unlock()
+	if !configured {
+		out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_not_configured", Object: "jarl",
+			Message: "sets take their addresses from Jarl, but housecarl.yaml names no Jarl server (jarl.server, docs/jarl.md): they keep the addresses they last had, none if Jarl was never read"})
+	} else if readErr != "" {
+		out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_unreachable", Object: "jarl",
+			Message: "reading Jarl failed: " + readErr + "; the Jarl sets keep the addresses they had"})
+	}
+	for _, s := range sets {
+		obj := "set " + s.Name
+		for _, n := range r.Status(st, s).Names {
+			switch {
+			case n.Problem != "":
+				out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_name", Object: obj,
+					Message: n.Name + viewSuffix(n.View) + ": " + n.Problem + "; it adds no addresses"})
+			case n.Error != "" && readErr == "":
+				out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_unreachable", Object: obj,
+					Message: n.Name + viewSuffix(n.View) + ": reading it from Jarl failed: " + n.Error + "; it keeps the addresses it had"})
+			}
+			for _, a := range n.Addresses {
+				if !a.Used {
+					out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_outside", Object: obj,
+						Message: fmt.Sprintf("%s%s has address %s, which is left out: %s", n.Name, viewSuffix(n.View), a.Address, a.Why)})
+				}
+			}
+		}
+	}
+	return out
+}
+
+func viewSuffix(v string) string {
+	if v == "" {
+		return ""
+	}
+	return "@" + v
+}

internal/jarlsets/jarlsets_test.go

@@ -0,0 +1,317 @@
+package jarlsets
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/netip"
+	"reflect"
+	"strings"
+	"testing"
+
+	"housecarl/internal/jarl"
+	"housecarl/internal/model"
+)
+
+// fakeJarl is a Jarl with a few zones.
+type fakeJarl struct {
+	zones   []jarl.ZoneInfo
+	records map[int64][]jarl.Record
+	down    bool           // AllZones fails
+	broken  map[int64]bool // Records fails
+	calls   int
+}
+
+func (f *fakeJarl) AllZones(ctx context.Context) ([]jarl.ZoneInfo, error) {
+	if f.down {
+		return nil, errors.New("can't reach Jarl")
+	}
+	return f.zones, nil
+}
+
+func (f *fakeJarl) Records(ctx context.Context, id int64) ([]jarl.Record, error) {
+	f.calls++
+	if f.broken[id] {
+		return nil, errors.New("Jarl answered 500")
+	}
+	return f.records[id], nil
+}
+
+func rec(name, typ, data string) jarl.Record { return jarl.Record{Name: name, Type: typ, Data: data} }
+
+// newFake: example.com in views internal (1) and external (2), linked
+// sub-zone wg.example.com in internal (3), other.org only in internal (4),
+// a secondary (5), a disabled zone (6), an in-view zone in vps (7).
+func newFake() *fakeJarl {
+	return &fakeJarl{
+		zones: []jarl.ZoneInfo{
+			{ID: 1, Name: "example.com.", View: "internal", Type: "primary"},
+			{ID: 2, Name: "example.com.", View: "external", Type: "primary"},
+			{ID: 3, Name: "wg.example.com.", View: "internal", Type: "primary"},
+			{ID: 4, Name: "other.org.", View: "internal", Type: "primary"},
+			{ID: 5, Name: "sec.net.", View: "internal", Type: "secondary"},
+			{ID: 6, Name: "off.net.", View: "internal", Type: "primary", Disabled: true},
+			{ID: 7, Name: "other.org.", View: "vps", Type: "in-view", InView: "internal"},
+			{ID: 8, Name: ".", View: "internal", Type: "hint"},
+		},
+		records: map[int64][]jarl.Record{
+			1: {
+				rec("example.com.", "NS", "ns1.example.com."),
+				rec("nas.example.com.", "A", "10.10.0.20"),
+				rec("nas.example.com.", "A", "10.10.0.21"),
+				rec("nas.example.com.", "AAAA", "2001:db8:10::20"),
+				rec("files.example.com.", "CNAME", "nas.example.com."),
+				rec("vpn.example.com.", "CNAME", "peer1.wg.example.com."),
+				rec("ext.example.com.", "CNAME", "host.duckdns.org."),
+				rec("loop1.example.com.", "CNAME", "loop2.example.com."),
+				rec("loop2.example.com.", "CNAME", "loop1"),
+				rec("rel.example.com.", "CNAME", "nas"),
+				rec("mail.example.com.", "MX", "10 nas.example.com."),
+				rec("lab.example.com.", "NS", "ns.lab.example.com."),
+				rec("www.example.com.", "A", "10.10.0.30"),
+				rec("bad.example.com.", "A", "127.0.0.1"),
+				rec("far.example.com.", "A", "192.0.2.9"),
+				rec("dead.example.com.", "CNAME", "nothing.example.com."),
+				rec("toother.example.com.", "CNAME", "srv.other.org."),
+			},
+			2: {
+				rec("www.example.com.", "A", "203.0.113.30"),
+			},
+			3: {
+				rec("peer1.wg.example.com.", "A", "10.50.50.2"),
+			},
+			4: {
+				rec("srv.other.org.", "A", "10.10.0.40"),
+			},
+		},
+		broken: map[int64]bool{},
+	}
+}
+
+func resolveOne(t *testing.T, f *fakeJarl, s string) *Lookup {
+	t.Helper()
+	l := &lookup{ctx: context.Background(), src: f, zones: f.zones, records: map[int64][]jarl.Record{}, errs: map[int64]error{}}
+	return l.resolve(model.ParseJarlName(s))
+}
+
+func TestResolve(t *testing.T) {
+	f := newFake()
+	cases := []struct {
+		name    string
+		addrs   []string
+		chain   []string
+		problem string // part of it
+		found   string
+	}{
+		{name: "nas.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, found: "internal"},
+		{name: "NAS.Example.com.@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, found: "internal"},
+		{name: "files.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, chain: []string{"nas.example.com"}},
+		{name: "rel.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, chain: []string{"nas.example.com"}},
+		// Into the sub-zone, and into another zone of the same view.
+		{name: "vpn.example.com@internal", addrs: []string{"10.50.50.2"}, chain: []string{"peer1.wg.example.com"}},
+		{name: "toother.example.com@internal", addrs: []string{"10.10.0.40"}, chain: []string{"srv.other.org"}},
+		// Only internal has wg.example.com, but external's example.com
+		// answers for the name too (that it doesn't exist): the view must be
+		// named.
+		{name: "peer1.wg.example.com", problem: "views external, internal"},
+		{name: "srv.other.org", problem: "views internal, vps"},
+		// Views give different answers.
+		{name: "www.example.com@internal", addrs: []string{"10.10.0.30"}},
+		{name: "www.example.com@external", addrs: []string{"203.0.113.30"}},
+		{name: "www.example.com", problem: "views external, internal: name the view"},
+		{name: "www.example.com@vps", problem: "no zone in view vps holds www.example.com"},
+		// An in-view zone serves the other view's records.
+		{name: "srv.other.org@vps", addrs: []string{"10.10.0.40"}},
+		{name: "ext.example.com@internal", chain: []string{"host.duckdns.org"}, problem: "its CNAME leads to host.duckdns.org, which Housecarl can't follow"},
+		{name: "loop1.example.com@internal", problem: "loop"},
+		{name: "dead.example.com@internal", chain: []string{"nothing.example.com"}, problem: "nothing.example.com, which has no A or AAAA record"},
+		{name: "mail.example.com@internal", problem: "has no A, AAAA or CNAME record"},
+		{name: "missing.example.com@internal", problem: "has no A, AAAA or CNAME record"},
+		{name: "host.lab.example.com@internal", problem: "below a delegation (NS records at lab.example.com)"},
+		{name: "a.sec.net", problem: "is a secondary zone"},
+		{name: "a.off.net", problem: "is disabled"},
+		{name: "nowhere.test", problem: "no zone in Jarl holds nowhere.test"},
+	}
+	for _, c := range cases {
+		lu := resolveOne(t, f, c.name)
+		if lu.Error != "" {
+			t.Errorf("%s: error %s", c.name, lu.Error)
+		}
+		if !reflect.DeepEqual(lu.Addresses, c.addrs) {
+			t.Errorf("%s: addresses %v, want %v (problem %q)", c.name, lu.Addresses, c.addrs, lu.Problem)
+		}
+		if c.chain != nil && !reflect.DeepEqual(lu.Chain, c.chain) {
+			t.Errorf("%s: chain %v, want %v", c.name, lu.Chain, c.chain)
+		}
+		if c.problem == "" && lu.Problem != "" || !strings.Contains(lu.Problem, c.problem) {
+			t.Errorf("%s: problem %q, want %q", c.name, lu.Problem, c.problem)
+		}
+		if c.found != "" && lu.Found != c.found {
+			t.Errorf("%s: found in %q, want %q", c.name, lu.Found, c.found)
+		}
+	}
+}
+
+func TestResolveNoViews(t *testing.T) {
+	f := &fakeJarl{zones: []jarl.ZoneInfo{{ID: 1, Name: "lan.", Type: "primary"}},
+		records: map[int64][]jarl.Record{1: {rec("nas.lan.", "A", "192.168.1.5")}}}
+	if lu := resolveOne(t, f, "nas.lan"); !reflect.DeepEqual(lu.Addresses, []string{"192.168.1.5"}) || lu.Problem != "" {
+		t.Errorf("%+v", lu)
+	}
+}
+
+// harness is a resolver over the fake Jarl, with lan = 10.10.0.0/24 and
+// wg = 10.50.50.0/24 connected.
+type harness struct {
+	r       *Resolver
+	f       *fakeJarl
+	st      *model.State
+	conn    map[netip.Prefix]string
+	applies int
+	audits  []string
+	saved   string
+}
+
+func newHarness(sets ...*model.AddressSet) *harness {
+	h := &harness{f: newFake(), conn: map[netip.Prefix]string{
+		netip.MustParsePrefix("10.10.0.0/24"):  "lan",
+		netip.MustParsePrefix("10.50.50.0/24"): "wg",
+	}}
+	h.st = &model.State{Zones: []*model.Zone{{Name: "lan"}, {Name: "wg"}, {Name: "wan"}}, Sets: sets}
+	h.r = &Resolver{Source: h.f, State: func() *model.State { return h.st }, Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
+		Connected: func(*model.State) map[netip.Prefix]string { return h.conn },
+		Apply: func(ctx context.Context) error {
+			h.applies++
+			h.r.Applied(h.r.Entries(h.st))
+			return nil
+		},
+		Audit: func(ctx context.Context, set, detail string) { h.audits = append(h.audits, set+": "+detail) },
+		Load:  func(ctx context.Context) (string, error) { return h.saved, nil },
+		Save:  func(ctx context.Context, v string) error { h.saved = v; return nil }}
+	return h
+}
+
+func jset(name string, within []string, names ...string) *model.AddressSet {
+	j := &model.JarlSource{Within: within}
+	for _, n := range names {
+		j.Names = append(j.Names, model.ParseJarlName(n))
+	}
+	return &model.AddressSet{Name: name, Jarl: j}
+}
+
+func TestWithin(t *testing.T) {
+	h := newHarness(
+		jset("nas", []string{"lan"}, "nas.example.com@internal", "vpn.example.com@internal", "bad.example.com@internal", "far.example.com@internal"),
+		jset("wide", []string{"lan", "wg", "192.0.2.0/24", "2001:db8:10::/64"}, "nas.example.com@internal", "vpn.example.com@internal", "far.example.com@internal"),
+		jset("none", []string{"lan"}, "missing.example.com@internal"),
+	)
+	h.r.Refresh(context.Background())
+	got := h.r.Entries(h.st)
+	want := map[string][]string{
+		"nas":  {"10.10.0.20", "10.10.0.21"},
+		"wide": {"10.10.0.20", "10.10.0.21", "10.50.50.2", "192.0.2.9", "2001:db8:10::20"},
+		"none": {},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("entries %v, want %v", got, want)
+	}
+	st := h.r.Status(h.st, h.st.Sets[0])
+	why := map[string]string{}
+	for _, n := range st.Names {
+		for _, a := range n.Addresses {
+			why[a.Address] = a.Why
+		}
+	}
+	if why["10.10.0.20"] != "" || !strings.Contains(why["10.50.50.2"], "not within lan") || !strings.Contains(why["127.0.0.1"], "loopback") || why["2001:db8:10::20"] == "" {
+		t.Errorf("why: %v", why)
+	}
+	codes := map[string]int{}
+	for _, p := range h.r.Problems(h.st) {
+		codes[p.Code]++
+	}
+	// nas: 3 left out (v6, wg, loopback, far = 4); wide: none; none: a name problem.
+	if codes["jarl_outside"] != 4 || codes["jarl_name"] != 1 || codes["jarl_unreachable"] != 0 {
+		t.Errorf("problems %v: %+v", codes, h.r.Problems(h.st))
+	}
+	// The lan interface loses its address: its networks as last seen count.
+	h.conn = map[netip.Prefix]string{}
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.20", "10.10.0.21"}) {
+		t.Errorf("without lan's address: %v", got)
+	}
+}
+
+func TestKeepsWhatItHad(t *testing.T) {
+	h := newHarness(jset("nas", []string{"lan", "wg"}, "nas.example.com@internal", "peer1.wg.example.com@internal"))
+	ctx := context.Background()
+	h.r.Refresh(ctx)
+	if h.applies != 1 || len(h.audits) != 1 || !strings.Contains(h.audits[0], "added 10.10.0.20, 10.10.0.21, 10.50.50.2") {
+		t.Fatalf("first read: %d applies, audits %v", h.applies, h.audits)
+	}
+	// Nothing changed: no apply.
+	h.r.Refresh(ctx)
+	if h.applies != 1 {
+		t.Errorf("unchanged: %d applies", h.applies)
+	}
+	// Jarl down: the addresses stay, a warning says so.
+	h.f.down = true
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; len(got) != 3 || h.applies != 1 {
+		t.Errorf("Jarl down: %v, %d applies", got, h.applies)
+	}
+	if ps := h.r.Problems(h.st); len(ps) != 2 || ps[0].Code != "jarl_unreachable" || ps[1].Code != "jarl_outside" {
+		t.Errorf("Jarl down: %+v", ps)
+	}
+	// One zone's records fail: only its name keeps the old addresses.
+	h.f.down = false
+	h.f.broken[3] = true
+	h.f.records[1][1] = rec("nas.example.com.", "A", "10.10.0.22")
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22", "10.50.50.2"}) {
+		t.Errorf("one zone down: %v", got)
+	}
+	if h.applies != 2 || !strings.Contains(h.audits[1], "added 10.10.0.22; removed 10.10.0.20") {
+		t.Errorf("audits %v", h.audits)
+	}
+	// After a restart, before Jarl is read: what was saved.
+	r2 := &Resolver{Load: h.r.Load, State: h.r.State, Connected: h.r.Connected}
+	r2.Restore(ctx)
+	if got := r2.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22", "10.50.50.2"}) {
+		t.Errorf("restored: %v", got)
+	}
+	// Jarl answers that the name is gone: its addresses go.
+	h.f.broken[3] = false
+	h.f.records[3] = nil
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22"}) {
+		t.Errorf("deleted record: %v", got)
+	}
+	// A name taken out of the set is forgotten.
+	h.st.Sets[0].Jarl.Names = h.st.Sets[0].Jarl.Names[:1]
+	h.r.Refresh(ctx)
+	if len(h.r.data.Lookups) != 1 {
+		t.Errorf("lookups %v", h.r.data.Lookups)
+	}
+}
+
+func TestNotConfigured(t *testing.T) {
+	h := newHarness(jset("nas", []string{"lan"}, "nas.example.com@internal"))
+	h.r.Source = nil
+	h.r.Refresh(context.Background())
+	if ps := h.r.Problems(h.st); len(ps) != 1 || ps[0].Code != "jarl_not_configured" {
+		t.Errorf("%+v", ps)
+	}
+	if got := h.r.Entries(h.st); !reflect.DeepEqual(got, map[string][]string{"nas": {}}) {
+		t.Errorf("%v", got)
+	}
+}
+
+// TestNoSetsNoApply: without Jarl sets nothing is applied, not even at the
+// first read (an empty configuration must never be applied).
+func TestNoSetsNoApply(t *testing.T) {
+	h := newHarness()
+	h.r.Refresh(context.Background())
+	if h.applies != 0 {
+		t.Errorf("%d applies", h.applies)
+	}
+}

internal/matrix/matrix.go

@@ -241,7 +241,7 @@ func verdictOf(action string) string {
 
 // elsewhere reports whether every address of refs (addresses, sets) lies
 // in networks connected to zones other than zone: then traffic of zone
-// can't have them. An empty list, a dynamic set, or an address the
+// can't have them. An empty list, a dynamic or Jarl set, or an address the
 // firewall isn't directly connected to (a routed network) might.
 func elsewhere(st *model.State, o Options, refs []string, zone string) bool {
 	if len(refs) == 0 || len(o.Connected) == 0 || zone == Firewall {
@@ -250,7 +250,7 @@ func elsewhere(st *model.State, o Options, refs []string, zone string) bool {
 	var entries []string
 	for _, r := range refs {
 		if s := st.Set(r); s != nil {
-			if s.Dynamic {
+			if s.Runtime() {
 				return false
 			}
 			entries = append(entries, s.Entries...)

internal/model/objects.go

@@ -1,6 +1,9 @@
 package model
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 // Chains rules are placed in.
 const (
@@ -71,11 +74,49 @@ type AddressSet struct {
 	Dynamic bool     `json:"dynamic,omitempty"`
 	// Timeout is a dynamic set's default entry lifetime in seconds (0:
 	// entries stay until removed).
-	Timeout int    `json:"timeout,omitempty"`
-	Comment string `json:"comment,omitempty"`
+	Timeout int `json:"timeout,omitempty"`
+	// Jarl makes the set's entries the addresses of names in Jarl, which
+	// housecarld reads and keeps up to date (docs/jarl.md).
+	Jarl    *JarlSource `json:"jarl,omitempty"`
+	Comment string      `json:"comment,omitempty"`
 }
 
-func (s *AddressSet) GetID() int64  { return s.ID }
+func (s *AddressSet) GetID() int64 { return s.ID }
+
+// Runtime reports whether the set's entries come at run time (a dynamic
+// set's, a Jarl set's) rather than from the configuration.
+func (s *AddressSet) Runtime() bool { return s.Dynamic || s.Jarl != nil }
+
+// JarlSource is where a set's addresses come from in Jarl: the A and AAAA
+// records of names (CNAMEs followed), each in a view, kept only where
+// Within allows.
+type JarlSource struct {
+	Names []JarlName `json:"names"`
+	// Within are zone names and addresses (prefixes, ranges): an address
+	// is used only if it lies in a zone's connected networks or in one of
+	// the addresses.
+	Within []string `json:"within"`
+}
+
+// JarlName is a name in Jarl, in a view ("": the only view that has it).
+type JarlName struct {
+	Name string `json:"name"` // "nas.example.com", without the final dot
+	View string `json:"view,omitempty"`
+}
+
+// String is "name" or "name@view".
+func (n JarlName) String() string {
+	if n.View == "" {
+		return n.Name
+	}
+	return n.Name + "@" + n.View
+}
+
+// ParseJarlName reads "name" or "name@view".
+func ParseJarlName(s string) JarlName {
+	name, view, _ := strings.Cut(strings.TrimSpace(s), "@")
+	return JarlName{Name: strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), "."), View: strings.TrimSpace(view)}
+}
 func (s *AddressSet) Label() string { return s.Name }
 
 // Service is a named list of ports ("tcp/22", "udp/60000-61000") and ICMP

internal/model/resolve.go

@@ -1,7 +1,8 @@
 package model
 
 // FamilyRefs are a reference list's part for one address family: names of
-// sets that have entries of the family (dynamic sets always do), and
+// sets that have entries of the family (dynamic and Jarl sets always
+// do), and
 // literal entries.
 type FamilyRefs struct {
 	Sets     []string
@@ -38,10 +39,10 @@ func (st *State) SplitRefs(refs []string) (v4, v6 FamilyRefs) {
 	return v4, v6
 }
 
-// Families reports which families a set holds: a dynamic set may get
-// either at run time.
+// Families reports which families a set holds: a dynamic or Jarl set may
+// get either at run time.
 func (s *AddressSet) Families() (v4, v6 bool) {
-	if s.Dynamic {
+	if s.Runtime() {
 		return true, true
 	}
 	for _, e := range s.Entries {

internal/nft/nft.go

@@ -455,11 +455,21 @@ type Firewall struct {
 	// Bans returns the active bans, which every apply puts back (the
 	// database keeps them; nil: none).
 	Bans func() []render.Ban
+	// JarlSets gives the Jarl sets' addresses (nil: none).
+	JarlSets JarlSets
 	// flowDevs are the flowtable's devices in the ruleset in force, by
 	// name, with their interface index then (0: left out, missing).
 	flowDevs map[string]int
 }
 
+// JarlSets are the Jarl sets' addresses (internal/jarlsets).
+type JarlSets interface {
+	// Entries are the addresses for st's Jarl sets: set name → addresses.
+	Entries(st *model.State) map[string][]string
+	// Applied is told what an apply put in force.
+	Applied(entries map[string][]string)
+}
+
 // ifIndex is an interface's index, 0 if it doesn't exist (tests change it).
 var ifIndex = func(name string) int {
 	if i, err := net.InterfaceByName(name); err == nil {
@@ -627,21 +637,24 @@ func (f *Firewall) ResetQuota(ctx context.Context, ruleID int64, perAddress bool
 	return nil
 }
 
-func (f *Firewall) options() render.Options {
+func (f *Firewall) options(st *model.State) render.Options {
 	o := render.Options{Shadow: f.Shadow, ShadowBefore: f.ShadowBefore}
 	if f.Interfaces != nil {
 		o.Interfaces = f.Interfaces()
 	}
+	if f.JarlSets != nil {
+		o.JarlSets = f.JarlSets.Entries(st)
+	}
 	return o
 }
 
 // Render builds the ruleset for st as Apply would.
-func (f *Firewall) Render(st *model.State) *render.Ruleset { return render.Render(st, f.options()) }
+func (f *Firewall) Render(st *model.State) *render.Ruleset { return render.Render(st, f.options(st)) }
 
 // RenderEnforcing builds the ruleset for st as Apply would outside shadow
 // mode.
 func (f *Firewall) RenderEnforcing(st *model.State) *render.Ruleset {
-	o := f.options()
+	o := f.options(st)
 	o.Shadow, o.ShadowBefore = false, false
 	return render.Render(st, o)
 }
@@ -649,7 +662,7 @@ func (f *Firewall) RenderEnforcing(st *model.State) *render.Ruleset {
 // Check implements engine.Firewall. In shadow mode it checks the
 // enforcing ruleset too, so that leaving shadow mode can't fail.
 func (f *Firewall) Check(ctx context.Context, st *model.State) error {
-	if err := f.Runner.Check(ctx, render.Render(st, f.options())); err != nil {
+	if err := f.Runner.Check(ctx, render.Render(st, f.options(st))); err != nil {
 		return err
 	}
 	if f.Shadow {
@@ -664,7 +677,7 @@ func (f *Firewall) Check(ctx context.Context, st *model.State) error {
 func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	opt := f.options()
+	opt := f.options(st)
 	var old *Table
 	if data, err := f.Runner.ListTable(ctx); err == nil {
 		if t, err := ParseTable(data); err == nil {
@@ -690,6 +703,9 @@ func (f *Firewall) Apply(ctx context.Context, st *model.State) error {
 	if err := f.Runner.Apply(ctx, render.Render(st, opt)); err != nil {
 		return err
 	}
+	if f.JarlSets != nil {
+		f.JarlSets.Applied(opt.JarlSets)
+	}
 	f.restore = nil
 	f.quotaRules = map[int64]*model.Rule{}
 	for _, r := range st.Rules {
@@ -778,7 +794,13 @@ func (f *Firewall) SaveBoot(st *model.State) error {
 	if err := writeAtomic(FallbackFile(f.BootFile), fallback); err != nil {
 		return err
 	}
-	rs := render.Render(st, render.Options{Interfaces: map[string]bool{}, Shadow: f.Shadow, ShadowBefore: f.ShadowBefore})
+	opt := render.Options{Interfaces: map[string]bool{}, Shadow: f.Shadow, ShadowBefore: f.ShadowBefore}
+	if f.JarlSets != nil {
+		// The addresses last read from Jarl: Jarl may not be reachable yet
+		// at boot.
+		opt.JarlSets = f.JarlSets.Entries(st)
+	}
+	rs := render.Render(st, opt)
 	return writeAtomic(f.BootFile, rs.Text)
 }
 

internal/nft/nft_test.go

@@ -131,6 +131,60 @@ func TestFirewallInContainer(t *testing.T) {
 	}
 }
 
+// fakeJarlSets gives fixed Jarl set addresses.
+type fakeJarlSets struct {
+	entries, applied map[string][]string
+}
+
+func (f *fakeJarlSets) Entries(*model.State) map[string][]string { return f.entries }
+func (f *fakeJarlSets) Applied(e map[string][]string)            { f.applied = e }
+
+// TestJarlSetsInContainer: a Jarl set gets its addresses with each apply,
+// and the new ones replace the old.
+func TestJarlSetsInContainer(t *testing.T) {
+	image := os.Getenv("HOUSECARL_NFT_IMAGE")
+	if image == "" {
+		t.Skip("HOUSECARL_NFT_IMAGE not set")
+	}
+	name := "housecarl-nft-jarl-test"
+	exec.Command("docker", "rm", "-f", name).Run()
+	if out, err := exec.Command("docker", "run", "-d", "--name", name, "--cap-add", "NET_ADMIN", image, "sleep", "300").CombinedOutput(); err != nil {
+		t.Fatalf("starting the container: %v %s", err, out)
+	}
+	t.Cleanup(func() { exec.Command("docker", "rm", "-f", name).Run() })
+	js := &fakeJarlSets{entries: map[string][]string{"nas": {"10.0.0.5", "10.0.0.6", "2001:db8::5"}}}
+	fw := &Firewall{Runner: &Runner{Command: []string{"docker", "exec", "-i", name, "nft"}}, JarlSets: js}
+	ctx := context.Background()
+	st := model.NewState()
+	st.Zones = []*model.Zone{{ID: 1, Name: "lan", Interfaces: []string{"eth0"}}}
+	st.Sets = []*model.AddressSet{{ID: 2, Name: "nas", Jarl: &model.JarlSource{Names: []model.JarlName{{Name: "nas.example.com"}}, Within: []string{"lan"}}}}
+	st.Rules = []*model.Rule{{ID: 3, Chain: model.ChainOutput, Position: 1, Action: model.ActionAccept, Match: model.Match{Destinations: []string{"nas"}}}}
+	elems := func() string {
+		tb, err := fw.Table(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got []string
+		for _, e := range tb.Elements["nas"] {
+			got = append(got, e.Addr)
+		}
+		return strings.Join(got, ",")
+	}
+	if err := fw.Apply(ctx, st); err != nil {
+		t.Fatalf("apply: %v", err)
+	}
+	if got := elems(); got != "10.0.0.5,10.0.0.6,2001:db8::5" || len(js.applied["nas"]) != 3 {
+		t.Errorf("after apply: %s (applied %v)", got, js.applied)
+	}
+	js.entries = map[string][]string{"nas": {"10.0.0.7"}}
+	if err := fw.Apply(ctx, st); err != nil {
+		t.Fatalf("reapply: %v", err)
+	}
+	if got := elems(); got != "10.0.0.7" {
+		t.Errorf("after the addresses changed: %s", got)
+	}
+}
+
 // TestQuotasInContainer: quotas keep what they used across a reload,
 // reset by hand, and come back after a reboot from what was saved.
 func TestQuotasInContainer(t *testing.T) {

internal/render/render.go

@@ -52,6 +52,9 @@ type Options struct {
 	DeviceSets map[string][]Element
 	// Bans are the ban lists' active bans, added to their sets.
 	Bans []Ban
+	// JarlSets are the Jarl sets' addresses (set name → addresses, both
+	// families), as Jarl gave them and within allows.
+	JarlSets map[string][]string
 	// Shadow renders for shadow mode: nothing is blocked, rejected,
 	// translated or changed (see the package comment).
 	Shadow bool
@@ -186,6 +189,10 @@ func (r *renderer) sets() {
 			}
 			continue
 		}
+		if s.Jarl != nil {
+			r.jarlSet(s)
+			continue
+		}
 		has4, has6 := s.Families()
 		for _, v6 := range []bool{false, true} {
 			if v6 && !has6 || !v6 && !has4 {
@@ -197,6 +204,26 @@ func (r *renderer) sets() {
 	}
 }
 
+// jarlSet declares a Jarl set's two sets with the addresses it has now
+// (none yet: empty, so its rules match nothing).
+func (r *renderer) jarlSet(s *model.AddressSet) {
+	for _, v6 := range []bool{false, true} {
+		var elems []string
+		for _, e := range r.opt.JarlSets[s.Name] {
+			if a, err := model.ParseAddress(e); err == nil && a.IsV6() == v6 {
+				elems = append(elems, a.String())
+			}
+		}
+		if len(elems) == 0 {
+			r.w.emit("set "+s.Name, "\tset %s { type %s; }", setName(s.Name, v6), addrType(v6))
+			continue
+		}
+		// Single addresses: no interval flag, so they stay as they are
+		// (auto-merge would join neighbours into ranges).
+		r.w.emit("set "+s.Name, "\tset %s { type %s; elements = { %s } }", setName(s.Name, v6), addrType(v6), strings.Join(elems, ", "))
+	}
+}
+
 func (r *renderer) elements() {
 	names := make([]string, 0, len(r.opt.Elements))
 	for n := range r.opt.Elements {

internal/render/render_test.go

@@ -35,6 +35,8 @@ func router() *model.State {
 		{ID: next(), Name: "wg_admins", Entries: []string{"10.50.50.3", "10.50.50.6"}},
 		{ID: next(), Name: "dns_servers", Entries: []string{"10.0.10.5", "217.155.8.194", "fdab:abab:abab:10::5", "2a02:8013:7f3d:10::5"}},
 		{ID: next(), Name: "guests", Dynamic: true, Timeout: 3600},
+		{ID: next(), Name: "nas", Jarl: &model.JarlSource{Names: []model.JarlName{{Name: "nas.example.com", View: "internal"}}, Within: []string{"lan"}}},
+		{ID: next(), Name: "printers", Jarl: &model.JarlSource{Names: []model.JarlName{{Name: "printer.example.com"}}, Within: []string{"lan"}}},
 	}
 	st.Services = []*model.Service{
 		{ID: next(), Name: "jarl", Entries: []string{"tcp/8053"}},
@@ -67,6 +69,7 @@ func router() *model.State {
 	rule(model.ChainForward, model.Rule{Match: model.Match{InZones: []string{"wan"}}, Action: model.ActionDrop, Comment: "nothing else in"})
 	rule(model.ChainForward, model.Rule{Match: model.Match{Destinations: []string{"wg_net"}}, Action: model.ActionDrop})
 	rule(model.ChainForward, model.Rule{Match: model.Match{Services: []string{"ping"}}})
+	rule(model.ChainForward, model.Rule{Match: model.Match{InZones: []string{"wg"}, Destinations: []string{"nas", "printers"}}, Comment: "from Jarl"})
 	rule(model.ChainForward, model.Rule{Match: model.Match{Sources: []string{"wg_net", "trusted", "guests"}}})
 	// Bandwidth: a byte rate, per destination; per /24; quotas, total and
 	// per device.
@@ -106,6 +109,9 @@ func router() *model.State {
 	return st
 }
 
+// jarlOptions are the Jarl sets' addresses.
+var jarlOptions = map[string][]string{"nas": {"10.0.10.20", "10.0.10.21", "2a02:8013:7f3d:10::20"}, "printers": {}}
+
 // banOptions are active bans in each ban list, of every port and of one.
 func banOptions(st *model.State) []Ban {
 	var out []Ban
@@ -137,13 +143,19 @@ func quotaOptions(st *model.State) (map[string]int64, map[string][]Element) {
 func TestRenderRouter(t *testing.T) {
 	st := router()
 	q, qs := quotaOptions(st)
-	rs := Render(st, Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}}}, Quotas: q, QuotaSets: qs})
+	rs := Render(st, Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}}}, Quotas: q, QuotaSets: qs, JarlSets: jarlOptions})
 	text := rs.Text
 	for _, want := range []string{
 		"table inet housecarl {}\ndelete table inet housecarl\ntable inet housecarl {",
 		`set trusted_v4 { type ipv4_addr; flags interval; auto-merge; elements = { 10.0.10.0/24, 10.20.30.0/24, 10.50.50.0/24 } }`,
 		`set trusted_v6 { type ipv6_addr; flags interval; auto-merge; elements = { 2a02:8013:7f3d:10::/64 } }`,
 		`set guests_v4 { type ipv4_addr; flags timeout; timeout 3600s; }`,
+		// Jarl sets: their addresses now, or empty.
+		`set nas_v4 { type ipv4_addr; elements = { 10.0.10.20, 10.0.10.21 } }`,
+		`set nas_v6 { type ipv6_addr; elements = { 2a02:8013:7f3d:10::20 } }`,
+		"set printers_v4 { type ipv4_addr; }\n",
+		`iifname "wg0" ip daddr @nas_v4 counter accept`,
+		`iifname "wg0" ip6 daddr @printers_v6 counter accept`,
 		`iifname "wan" ip saddr { 0.0.0.0/8,`,
 		`iifname "wan" ip6 saddr { ::1/128, ::/128, fc00::/7, 2001:db8::/32, ff00::/8, 2a02:8013:7f3d::/48 } counter drop`,
 		`oifname "wan" tcp flags syn tcp option maxseg size set rt mtu comment "hc:b-mss-clamp"`,
@@ -278,7 +290,7 @@ func TestRenderedRulesetPassesNft(t *testing.T) {
 		for _, mode := range []Options{{}, {Shadow: true}, {Shadow: true, ShadowBefore: true}} {
 			opt := Options{Elements: map[string][]Element{"guests": {{Addr: "10.0.30.5", Timeout: 1700}, {Addr: "2a02:8013:7f3d:30::5"}}}, Quotas: q, QuotaSets: qs, Bans: banOptions(st),
 				DeviceSets: map[string][]Element{"stats_up_v4": {{Addr: "10.0.10.5", Timeout: 3000, Packets: 7, Bytes: 7000}}, "stats_down_v6": {{Addr: "2001:db8:1:2::", Packets: 1, Bytes: 100}}},
-				Shadow:     mode.Shadow, ShadowBefore: mode.ShadowBefore}
+				JarlSets:   jarlOptions, Shadow: mode.Shadow, ShadowBefore: mode.ShadowBefore}
 			nftCheck(t, cmd, fmt.Sprintf("%s (shadow %v, before %v)", name, mode.Shadow, mode.ShadowBefore), Render(st, opt).Text)
 		}
 	}

internal/server/h_bans.go

@@ -394,7 +394,7 @@ func (s *Server) inNever(ctx context.Context, st *model.State, ref string, prefi
 		return ""
 	}
 	entries := set.Entries
-	if set.Dynamic {
+	if set.Runtime() {
 		if t, err := s.fw.Table(ctx); err == nil {
 			for _, e := range t.Elements[set.Name] {
 				entries = append(entries, e.Addr)

internal/server/h_config.go

@@ -668,6 +668,10 @@ func matrixHits(m *api.Matrix, counters map[string]nft.Counter) {
 
 // connected maps the networks on the firewall's interfaces (link-local
 // and loopback aside) to their zones.
+// Connected are the networks connected to the firewall's interfaces,
+// with their zones (link-local and loopback left out).
+func (s *Server) Connected(st *model.State) map[netip.Prefix]string { return s.connected(st) }
+
 func (s *Server) connected(st *model.State) map[netip.Prefix]string {
 	out := map[netip.Prefix]string{}
 	for _, i := range s.hostInterfaces(st) {

internal/server/h_status.go

@@ -84,16 +84,21 @@ var nftablesUnitLinks = []string{
 
 // hostProblems are warnings about the machine rather than the
 // configuration: another ruleset that still loads at boot, or the table
-// an import came from, still in force beside Housecarl's. ruleset is the
+// an import came from, still in force beside Housecarl's, and the Jarl
+// sets' names that give no addresses. ruleset is the
 // machine's (nft -j list ruleset; nil: not read). In shadow mode those
 // are expected; the warnings are that nothing else filters, and the raw
 // rules left out.
 func (s *Server) hostProblems(ctx context.Context, ruleset []byte) []validate.Problem {
+	var jarl []validate.Problem
+	if s.cfg.JarlSets != nil {
+		jarl = s.cfg.JarlSets.Problems(s.eng.State())
+	}
 	if s.cfg.Shadow {
-		return s.shadowProblems(ruleset)
+		return append(s.shadowProblems(ruleset), jarl...)
 	}
 	others := otherTables(ruleset)
-	var out []validate.Problem
+	out := jarl
 	for _, link := range s.unitLinks {
 		if _, err := os.Lstat(link); err == nil {
 			out = append(out, validate.Problem{Severity: validate.Warning, Code: "nftables_service", Object: "nftables.service",
@@ -284,6 +289,53 @@ func (s *Server) deleteElement(w http.ResponseWriter, r *http.Request, p *auth.P
 	return nil
 }
 
+// ---- Jarl sets
+
+// jarlSet is the Jarl set a request names.
+func (s *Server) jarlSet(r *http.Request, p *auth.Principal) (*model.State, *model.AddressSet, error) {
+	name := r.PathValue("ref")
+	if !p.CanSet(name, false) {
+		return nil, nil, forbidden("permission denied: this needs sets:read for %s", name)
+	}
+	st := s.eng.State()
+	set := st.Set(name)
+	if set == nil {
+		return nil, nil, notFound("no set %s", name)
+	}
+	if set.Jarl == nil {
+		return nil, nil, badRequest("set %s doesn't take its addresses from Jarl", set.Name)
+	}
+	if s.cfg.JarlSets == nil {
+		return nil, nil, conflict("this housecarld doesn't read Jarl sets")
+	}
+	return st, set, nil
+}
+
+// jarlStatus: GET /sets/{ref}/jarl.
+func (s *Server) jarlStatus(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, set, err := s.jarlSet(r, p)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, s.cfg.JarlSets.Status(st, set))
+	return nil
+}
+
+// jarlRefresh: POST /sets/{ref}/jarl/refresh reads Jarl now (every Jarl
+// set) and answers as GET does.
+func (s *Server) jarlRefresh(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if _, _, err := s.jarlSet(r, p); err != nil {
+		return err
+	}
+	s.cfg.JarlSets.Refresh(r.Context())
+	st, set, err := s.jarlSet(r, p)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, s.cfg.JarlSets.Status(st, set))
+	return nil
+}
+
 // ---- the log
 
 // logRing keeps the latest logged packets.

internal/server/jarlsets_test.go

@@ -0,0 +1,104 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"housecarl/internal/api"
+	"housecarl/internal/jarl"
+	"housecarl/internal/jarlsets"
+	"housecarl/internal/model"
+	"housecarl/internal/store"
+)
+
+// TestJarlSets: a set from Jarl, through Housecarl's Jarl client against a
+// fake Jarl, filled after a change, with its status and warnings.
+func TestJarlSets(t *testing.T) {
+	records := []map[string]string{
+		{"name": "nas.example.com.", "type": "A", "data": "10.0.10.20"},
+		{"name": "nas.example.com.", "type": "AAAA", "data": "2001:db8:10::20"},
+		{"name": "files.example.com.", "type": "CNAME", "data": "nas.example.com."},
+		{"name": "far.example.com.", "type": "A", "data": "192.0.2.9"},
+	}
+	fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer jarl-token" {
+			w.WriteHeader(401)
+			return
+		}
+		switch r.URL.Path {
+		case "/api/v1/zones":
+			json.NewEncoder(w).Encode([]map[string]any{{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"}})
+		case "/api/v1/zones/1/records":
+			if r.URL.Query().Get("disabled") != "0" {
+				t.Errorf("records asked without disabled=0: %s", r.URL)
+			}
+			json.NewEncoder(w).Encode(records)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	t.Cleanup(fake.Close)
+	jc, err := jarl.New(fake.URL, "jarl-token", "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	js := &jarlsets.Resolver{Source: jc}
+	e := newEnvWith(t, Config{JarlSets: js})
+	js.State, js.Connected = e.eng.State, e.srv.Connected
+	js.Apply = func(ctx context.Context) error { return e.eng.ApplyCurrent(ctx) }
+	js.Audit = func(ctx context.Context, set, detail string) {
+		e.eng.Audit(ctx, nil, "set.jarl", set, "success", detail)
+	}
+	ctx := context.Background()
+
+	e.call(e.token, "POST", "/zones", model.Zone{Name: "lan", Interfaces: []string{"eth1"}}, nil) // 10.0.10.0/24
+	set := model.AddressSet{Name: "nas", Jarl: &model.JarlSource{Names: []model.JarlName{{Name: "files.example.com"}, {Name: "far.example.com", View: "internal"}}, Within: []string{"lan"}}}
+	if code := e.call(e.token, "POST", "/sets", set, nil); code != 201 {
+		t.Fatalf("create: %d", code)
+	}
+	if code := e.call(e.token, "POST", "/sets", model.AddressSet{Name: "bad", Entries: []string{"10.0.0.1"}, Jarl: set.Jarl}, nil); code != 422 {
+		t.Errorf("entries in a Jarl set: %d", code)
+	}
+	var st api.JarlSet
+	if code := e.call(e.token, "POST", "/sets/nas/jarl/refresh", nil, &st); code != 200 {
+		t.Fatalf("refresh: %d", code)
+	}
+	if strings.Join(st.Entries, ",") != "10.0.10.20" || len(st.Names) != 2 || !st.Configured || st.Read == nil {
+		t.Fatalf("status %+v", st)
+	}
+	files := st.Names[0]
+	if files.Found != "internal" || strings.Join(files.Chain, ",") != "nas.example.com" || len(files.Addresses) != 2 ||
+		!files.Addresses[0].Used || files.Addresses[1].Used || !strings.Contains(files.Addresses[1].Why, "not within lan") {
+		t.Errorf("files: %+v", files)
+	}
+	var status api.Status
+	e.call(e.token, "GET", "/status", nil, &status)
+	n := 0
+	for _, p := range status.Problems {
+		if p.Code == "jarl_outside" {
+			n++
+		}
+	}
+	if n != 2 {
+		t.Errorf("jarl_outside warnings: %d (%+v)", n, status.Problems)
+	}
+	// A Jarl set can be renamed (its addresses go by its names in Jarl).
+	if code := e.call(e.token, "PATCH", "/sets/nas", map[string]any{"name": "storage"}, nil); code != 200 {
+		t.Errorf("rename: %d", code)
+	}
+	if code := e.call(e.token, "GET", "/sets/trusted/jarl", nil, nil); code != 404 {
+		t.Errorf("no such set: %d", code)
+	}
+	e.call(e.token, "POST", "/sets", model.AddressSet{Name: "trusted", Entries: []string{"10.0.10.0/24"}}, nil)
+	if code := e.call(e.token, "GET", "/sets/trusted/jarl", nil, nil); code != 400 {
+		t.Errorf("a set not from Jarl: %d", code)
+	}
+	evs, _ := e.srv.store.ListAudit(ctx, store.AuditFilter{Action: "set.jarl"})
+	if len(evs) != 1 || evs[0].Detail != "added 10.0.10.20" {
+		t.Errorf("audit %+v", evs)
+	}
+}

internal/server/routes.go

@@ -38,6 +38,8 @@ func (s *Server) routes() {
 	m.HandleFunc("GET "+p+"/sets/{ref}/elements", s.authed(s.listElements))
 	m.HandleFunc("POST "+p+"/sets/{ref}/elements", s.authed(s.addElement))
 	m.HandleFunc("DELETE "+p+"/sets/{ref}/elements/{addr}", s.authed(s.deleteElement))
+	m.HandleFunc("GET "+p+"/sets/{ref}/jarl", s.authed(s.jarlStatus))
+	m.HandleFunc("POST "+p+"/sets/{ref}/jarl/refresh", s.authed(s.jarlRefresh))
 	m.HandleFunc("GET "+p+"/bans", s.authed(s.listBans))
 	m.HandleFunc("GET "+p+"/banlists/{ref}/bans", s.authed(s.listBans))
 	m.HandleFunc("POST "+p+"/banlists/{ref}/bans", s.authed(s.addBan))

internal/server/server.go

@@ -22,6 +22,7 @@ import (
 	"housecarl/internal/auth"
 	"housecarl/internal/certs"
 	"housecarl/internal/engine"
+	"housecarl/internal/jarlsets"
 	"housecarl/internal/model"
 	"housecarl/internal/names"
 	"housecarl/internal/nft"
@@ -54,6 +55,8 @@ type Config struct {
 	ShadowPosition string
 	// Names looks up inside addresses' names (nil: names are off).
 	Names *names.Resolver
+	// JarlSets fills the sets whose addresses come from Jarl (nil: none).
+	JarlSets *jarlsets.Resolver
 }
 
 // Interfaces lists the host's interfaces with their addresses, and
@@ -153,6 +156,9 @@ func New(cfg Config, eng *engine.Engine, fw Firewall, ifaces Interfaces, log *sl
 		}
 		s.SaveQuotaUsage(context.Background())
 		s.pruneBans(context.Background(), st) // the engine is locked: not s.eng.State()
+		if cfg.JarlSets != nil {
+			cfg.JarlSets.Kick() // a new name: read it now
+		}
 	}
 	return s
 }

internal/validate/validate.go

@@ -247,15 +247,79 @@ func (c *checker) sets() {
 		if len(s.Entries) > 0 && s.Dynamic {
 			c.add(Error, "set_dynamic_entries", obj, "a dynamic set gets its entries at run time (hc set entries add, the Entries button), each with its timeout; leave out the configured ones, or make it a set that isn't dynamic")
 		}
-		if len(s.Entries) == 0 && !s.Dynamic {
+		if len(s.Entries) == 0 && !s.Runtime() {
 			c.add(Warning, "set_empty", obj, "the set is empty: rules using it match nothing")
 		}
+		if s.Jarl != nil {
+			c.jarlSet(obj, s)
+		}
 		if s.Timeout < 0 || s.Timeout > 0 && !s.Dynamic {
 			c.add(Error, "set_timeout", obj, "only a dynamic set has a timeout, of 1 second or more")
 		}
 	}
 }
 
+// dnsLabel is one label of a name in Jarl (underscores allowed, as in
+// _service names).
+var dnsLabel = regexp.MustCompile(`^[a-z0-9_]([a-z0-9_-]{0,61}[a-z0-9_])?$`)
+
+// jarlSet checks a set whose entries come from Jarl.
+func (c *checker) jarlSet(obj string, s *model.AddressSet) {
+	j := s.Jarl
+	if len(s.Entries) > 0 {
+		c.add(Error, "set_jarl_entries", obj, "a Jarl set's entries are the addresses of its names in Jarl; leave out the configured ones, or make it a set that doesn't come from Jarl")
+	}
+	if s.Dynamic {
+		c.add(Error, "set_jarl_dynamic", obj, "a set is dynamic or comes from Jarl, not both")
+	}
+	if len(j.Names) == 0 {
+		c.add(Error, "set_jarl_names", obj, "name at least one name in Jarl, such as nas.example.com (or nas.example.com@internal for one view)")
+	}
+	seen := map[string]bool{}
+	for _, n := range j.Names {
+		if !validDNSName(n.Name) {
+			c.add(Error, "set_jarl_names", obj, "%q isn't a DNS name; give the full name, such as nas.example.com (wildcards aren't followed)", n.Name)
+			continue
+		}
+		if strings.ContainsAny(n.View, " \t@") {
+			c.add(Error, "set_jarl_names", obj, "%q isn't a view name: write NAME@VIEW, such as nas.example.com@internal", n.View)
+			continue
+		}
+		if seen[n.String()] {
+			c.add(Error, "set_jarl_names", obj, "%s is listed twice", n)
+		}
+		seen[n.String()] = true
+	}
+	if len(j.Within) == 0 {
+		c.add(Error, "set_jarl_within", obj, "say where its addresses may be (within): zones, whose connected networks count, or networks such as 10.10.0.0/24; an address from Jarl elsewhere is left out, so a changed record can't open the firewall to anything")
+	}
+	for _, w := range j.Within {
+		if a, err := model.ParseAddress(w); err == nil {
+			if a.Prefix.IsValid() && a.Prefix.Bits() == 0 && !a.First.IsValid() {
+				c.add(Warning, "set_jarl_anywhere", obj, "within %s lets any address in: whoever can change the names in Jarl decides what the set holds; name zones or networks instead", w)
+			}
+			continue
+		}
+		if c.st.Zone(w) == nil {
+			c.add(Error, "set_jarl_within", obj, "within %q: no such zone, and not an address or network", w)
+		}
+	}
+}
+
+// validDNSName reports whether n is a name a Jarl set can look up: LDH
+// labels (and underscores), no wildcard, without the final dot.
+func validDNSName(n string) bool {
+	if n == "" || len(n) > 253 {
+		return false
+	}
+	for _, l := range strings.Split(n, ".") {
+		if !dnsLabel.MatchString(l) {
+			return false
+		}
+	}
+	return true
+}
+
 func (c *checker) services() {
 	seen := map[string]string{}
 	for _, s := range c.st.Services {

internal/validate/validate_test.go

@@ -29,6 +29,19 @@ func base() *model.State {
 	}
 }
 
+func jarlSource() *model.JarlSource {
+	return &model.JarlSource{Names: []model.JarlName{{Name: "nas.example.com", View: "internal"}}, Within: []string{"lan", "10.9.0.0/16"}}
+}
+
+func TestJarlSetValid(t *testing.T) {
+	st := base()
+	st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+	st.Sets[0].Jarl.Names = append(st.Sets[0].Jarl.Names, model.JarlName{Name: "_svc.files.example.com"})
+	if ps := Check(st, nil); len(ps) > 0 {
+		t.Fatalf("problems: %+v", ps)
+	}
+}
+
 func codes(ps []Problem) string {
 	var out []string
 	for _, p := range ps {
@@ -77,6 +90,33 @@ func TestCodes(t *testing.T) {
 		}},
 		{"set_timeout", func(st *model.State) { st.Sets[0].Timeout = 60 }},
 		{"set_dynamic_entries", func(st *model.State) { st.Sets[0].Dynamic = true }},
+		{"set_jarl_entries", func(st *model.State) { st.Sets[0].Jarl = jarlSource() }},
+		{"set_jarl_dynamic", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Dynamic, st.Sets[0].Jarl = nil, true, jarlSource()
+		}},
+		{"set_jarl_names", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, &model.JarlSource{Within: []string{"lan"}}
+		}},
+		{"set_jarl_names", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+			st.Sets[0].Jarl.Names[0].Name = "*.example.com"
+		}},
+		{"set_jarl_names", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+			st.Sets[0].Jarl.Names = append(st.Sets[0].Jarl.Names, st.Sets[0].Jarl.Names[0])
+		}},
+		{"set_jarl_within", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+			st.Sets[0].Jarl.Within = nil
+		}},
+		{"set_jarl_within", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+			st.Sets[0].Jarl.Within = []string{"dmz"}
+		}},
+		{"set_jarl_anywhere", func(st *model.State) {
+			st.Sets[0].Entries, st.Sets[0].Jarl = nil, jarlSource()
+			st.Sets[0].Jarl.Within = []string{"::/0"}
+		}},
 		{"service_empty", func(st *model.State) { st.Services[0].Entries = nil }},
 		{"service_entry", func(st *model.State) { st.Services[0].Entries = []string{"tcp/99999"} }},
 		{"unknown_zone", func(st *model.State) { st.Rules[1].InZones = []string{"dmz"} }},

internal/web/static/assets/pages/sets.js

@@ -1,5 +1,5 @@
-// Address sets, and dynamic sets' entries.
-import { h, mount, table, modal, errorBox, field, input, checkbox, confirmDialog, badge, toast } from "../ui.js";
+// Address sets, dynamic sets' entries, and what Jarl sets hold.
+import { h, mount, table, modal, errorBox, field, input, select, confirmDialog, badge, toast, fmtTime } from "../ui.js";
 import { get, api, change, can } from "../api.js";
 import { rerender } from "../app.js";
 import { listEditor, fmtSeconds, parseSeconds, rowClass } from "../widgets.js";
@@ -9,11 +9,14 @@ export async function render(el) {
   const write = can("config", true);
   mount(el,
     h("div", { class: "page-head" },
-      h("div", { class: "grow" }, h("h1", null, "Address sets"), h("div", { class: "muted" }, "Named lists of IPv4 and IPv6 addresses, prefixes and ranges, mixed. A dynamic set takes entries at run time, each with a timeout (a captive portal's signed-in clients).")),
+      h("div", { class: "grow" }, h("h1", null, "Address sets"), h("div", { class: "muted" }, "Named lists of IPv4 and IPv6 addresses, prefixes and ranges, mixed. A dynamic set takes entries at run time, each with a timeout (a captive portal's signed-in clients). A Jarl set holds the addresses of names in Jarl, kept up to date.")),
       write ? h("div", { class: "actions" }, h("button", { class: "primary", onclick: () => setDialog(null) }, "Add set")) : null),
     h("div", { class: "panel" }, table([
       { label: "Name", render: (s) => h("strong", null, s.name) },
-      { label: "Entries", render: (s) => s.dynamic ? [badge("dynamic", "accent"), s.timeout ? " timeout " + fmtSeconds(s.timeout) : "", " ",
+      { label: "Entries", render: (s) => s.jarl ? [badge("Jarl", "accent"), " ", h("span", { class: "mono small" }, jarlNames(s).join(", ")),
+        h("span", { class: "muted small" }, " within " + s.jarl.within.join(", ")), " ",
+        s.pending ? null : h("button", { class: "small", onclick: () => jarlStatus(s) }, "Addresses…")]
+        : s.dynamic ? [badge("dynamic", "accent"), s.timeout ? " timeout " + fmtSeconds(s.timeout) : "", " ",
         s.pending ? null : h("button", { class: "small", onclick: () => entries(s) }, "Entries…")] : h("span", { class: "mono small" }, s.entries.join(", ")) },
       { label: "Comment", render: (s) => s.comment },
       write ? { label: "", tdClass: "actions-cell", render: (s) => s.pending === "deleted" ? badge("deleted", "err") : [
@@ -27,33 +30,52 @@ async function del(s) {
   try { await change("DELETE", "/sets/" + s.id); rerender(); } catch (e) { modal("Not deleted", errorBox(e)); }
 }
 
+const nameOf = (n) => n.view ? n.name + "@" + n.view : n.name;
+const jarlNames = (s) => (s.jarl.names || []).map(nameOf);
+
+// parseJarlName reads "name" or "name@view".
+function parseJarlName(v) {
+  const i = v.indexOf("@");
+  return i < 0 ? { name: v } : { name: v.slice(0, i), view: v.slice(i + 1) };
+}
+
 async function setDialog(s) {
   s = s || { entries: [] };
+  const zones = (await get("/zones")).map((z) => z.name);
   const name = input({ value: s.name || "", placeholder: "trusted" });
   const list = listEditor(s.entries, { placeholder: "192.168.1.0/24, 2001:db8:1::/64, 192.168.1.100-192.168.1.199", addLabel: "Add entry" });
-  const dynamic = checkbox("dynamic: entries are added at run time (the Entries button, hc set entries, a program with a token), each for a time", s.dynamic);
-  // A dynamic set has no configured entries; only it has a timeout.
+  const kind = select([["list", "addresses listed here"], ["dynamic", "dynamic: entries added at run time, each for a time"], ["jarl", "from Jarl: the addresses of names in Jarl"]],
+    s.jarl ? "jarl" : s.dynamic ? "dynamic" : "list");
+  const names = listEditor(s.jarl ? jarlNames(s) : [], { placeholder: "nas.example.com@internal", addLabel: "Add name" });
+  const within = listEditor(s.jarl ? s.jarl.within : [], { placeholder: "lan, or 10.10.0.0/24", addLabel: "Add zone or network", suggestions: zones });
+  const timeout = input({ value: s.timeout ? fmtSeconds(s.timeout) : "", placeholder: "1h, 1d (none: until removed)", class: "short" });
+  // Each kind shows its own fields.
   const entryFields = field("Entries", list).filter(Boolean);
   const timeoutFields = field("Entry timeout", timeout, "How long an entry stays unless it is added with its own. Empty: until removed (or a reboot).").filter(Boolean);
+  const jarlFields = [
+    ...field("Names in Jarl", names, "The set holds their A and AAAA records' addresses (all of them), CNAMEs followed. Write NAME@VIEW, or just NAME if only one of Jarl's views has it. Read again every minute.").filter(Boolean),
+    ...field("Within", within, "Where its addresses may be: zones (the networks connected to their interfaces) or networks. An address elsewhere is left out, so a changed record in Jarl can't let anything else in.").filter(Boolean),
+  ];
   const showKind = () => {
-    for (const n of entryFields) n.classList.toggle("hidden", dynamic.input.checked);
-    for (const n of timeoutFields) n.classList.toggle("hidden", !dynamic.input.checked);
+    for (const n of entryFields) n.classList.toggle("hidden", kind.value !== "list");
+    for (const n of timeoutFields) n.classList.toggle("hidden", kind.value !== "dynamic");
+    for (const n of jarlFields) n.classList.toggle("hidden", kind.value !== "jarl");
   };
-  dynamic.input.addEventListener("change", showKind);
+  kind.addEventListener("change", showKind);
   showKind();
-  const timeout = input({ value: s.timeout ? fmtSeconds(s.timeout) : "", placeholder: "1h, 1d (none: until removed)", class: "short" });
   const comment = input({ value: s.comment || "" });
   const err = h("div");
   await modal(s.id ? "Edit set " + s.name : "Add set", h("div", null, h("div", { class: "form-grid" },
-    field("Name", name), ...entryFields, h("label"), dynamic, ...timeoutFields, field("Comment", comment)), err), [
+    field("Name", name), ...field("Kind", kind), ...entryFields, ...timeoutFields, ...jarlFields, field("Comment", comment)), err), [
     { label: "Cancel", value: null },
     { label: s.id ? "Save" : "Add set", kind: "primary", onclick: async () => {
       mount(err);
       const t = parseSeconds(timeout.value);
       if (t === null) { mount(err, h("div", { class: "msg err" }, "Write the timeout as 1h, 30m or 1d.")); return false; }
+      const k = kind.value;
       try {
-        await change(s.id ? "PATCH" : "POST", s.id ? "/sets/" + s.id : "/sets", { name: name.value.trim(), entries: dynamic.input.checked ? [] : list.value(), dynamic: dynamic.input.checked,
-          timeout: dynamic.input.checked ? t : 0, comment: comment.value.trim() });
+        await change(s.id ? "PATCH" : "POST", s.id ? "/sets/" + s.id : "/sets", { name: name.value.trim(), entries: k === "list" ? list.value() : [], dynamic: k === "dynamic",
+          timeout: k === "dynamic" ? t : 0, jarl: k === "jarl" ? { names: names.value().map(parseJarlName), within: within.value() } : null, comment: comment.value.trim() });
         rerender();
         return true;
       } catch (e) { mount(err, errorBox(e)); return false; }
@@ -61,6 +83,44 @@ async function setDialog(s) {
   ], { wide: true, sticky: true });
 }
 
+// jarlStatus shows what a Jarl set holds: each name's addresses in Jarl,
+// used or left out (and why).
+async function jarlStatus(s) {
+  const body = h("div");
+  const path = "/sets/" + encodeURIComponent(s.name) + "/jarl";
+  const draw = (js) => {
+    const rows = [];
+    for (const n of js.names) {
+      // A new node for each row: a node can only be in one place.
+      const label = () => [h("span", { class: "mono" }, n.name), n.chain && n.chain.length ? h("span", { class: "muted small" }, " → " + n.chain.join(" → ")) : null];
+      const view = n.view || n.found_view || "";
+      if (n.problem) rows.push({ label, view, address: "", used: h("span", null, badge("no addresses", "err"), " " + n.problem) });
+      else if (!n.addresses.length) rows.push({ label, view, address: "", used: n.error ? "not read: " + n.error : "not read yet" });
+      for (const a of n.addresses) rows.push({ label, view, address: a.address, used: a.used ? badge("used", "ok") : h("span", null, badge("left out", "warn"), " " + a.why) });
+      if (n.error && n.addresses.length) rows.push({ label, view, address: "", used: h("span", { class: "muted" }, "The last read failed (" + n.error + "): these are from " + (n.checked ? fmtTime(n.checked) : "before") + ".") });
+    }
+    mount(body,
+      !js.configured ? h("div", { class: "msg warn" }, "Jarl isn't configured (jarl.server in housecarl.yaml): the set keeps the addresses it last had.") : null,
+      js.error ? h("div", { class: "msg warn" }, "Reading Jarl failed: " + js.error + ". The set keeps the addresses it had.") : null,
+      h("p", null, "Holds ", h("span", { class: "mono" }, js.entries.length ? js.entries.join(", ") : "nothing"), " · within " + js.within.join(", "),
+        js.read ? h("span", { class: "muted" }, " · Jarl read " + fmtTime(js.read)) : null),
+      table([
+        { label: "Name", render: (r) => r.label() },
+        { label: "View", render: (r) => r.view },
+        { label: "Address", render: (r) => h("span", { class: "mono" }, r.address) },
+        { label: "", render: (r) => r.used },
+      ], rows, { empty: "No names." }));
+  };
+  draw(await get(path));
+  await modal("Addresses of " + s.name, body, [
+    { label: "Read Jarl now", onclick: async () => {
+      try { draw(await api("POST", path + "/refresh")); } catch (x) { toast(x.message, "err"); }
+      return false;
+    } },
+    { label: "Close", value: null },
+  ], { wide: true });
+}
+
 // entries shows and changes a dynamic set's entries.
 async function entries(s) {
   const body = h("div");

scripts/e2e.sh

@@ -20,7 +20,8 @@ IMAGE=${E2E_IMAGE:-housecarl-nft}
 WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
 docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
 
-(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
+(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc &&
+	CGO_ENABLED=0 go build -o "$WORK/fakejarl" ./scripts/fakejarl) || exit 1
 # The image: Debian 13 with nftables and network tools. 'docker build'
 # can't reach the network on some hosts, so it is made with run + commit.
 if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
@@ -104,6 +105,7 @@ ruleset: /w/data/ruleset.nft
 backup: {dir: /w/backups}
 log: {level: debug}
 stats: {database: /w/data/stats.db}
+jarl: {server: "http://127.0.0.1:8053", token: e2e-jarl-token, refresh: 10s}
 EOF
 { cat "$WORK/housecarl.yaml"; echo "shadow: {enabled: true}"; } > "$WORK/housecarl-shadow.yaml"
 CONF=housecarl.yaml
@@ -579,6 +581,77 @@ not_contains "dhcp "
 run "the ruleset is unchanged by it" 0 R nft list chain inet housecarl input
 contains 'iifname "enp2s0" udp dport 67'
 
+# ---- a set from Jarl: names' addresses, kept up to date
+
+# A fake Jarl on the router: example.com in views internal and external.
+# jf's AAAA is jellyfin's; ssh.example.com is a CNAME for it; the internet
+# host's address is outside the LAN.
+jarl_data() { # JF_AAAA
+	cat > "$WORK/jarl.json" <<EOJ
+{"zones": [{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"},
+           {"id": 2, "name": "example.com.", "view": "external", "type": "primary"}],
+ "records": {"1": [{"name": "jf.example.com.", "type": "A", "data": "$JF4"},
+                   {"name": "jf.example.com.", "type": "AAAA", "data": "$1"},
+                   {"name": "ssh.example.com.", "type": "CNAME", "data": "jf.example.com."},
+                   {"name": "net.example.com.", "type": "AAAA", "data": "$NET6"}],
+             "2": [{"name": "ssh.example.com.", "type": "AAAA", "data": "$NET6"}]}}
+EOJ
+}
+jarl_data $JF6
+docker exec -d ${P}-router sh -c '/w/fakejarl -listen 127.0.0.1:8053 -data /w/jarl.json -token e2e-jarl-token >> /w/fakejarl.log 2>&1'
+sleep 1
+run "IPv6 ssh to jellyfin is dropped" 1 connect net $JF6 22
+run "a set from Jarl" 0 H set add jfssh --jarl ssh.example.com --within lan
+run "a name in two views needs its view" 0 H set jarl jfssh --refresh
+contains "in Jarl's views external, internal: name the view"
+run "name the views" 0 H set update jfssh --jarl ssh.example.com@internal --jarl net.example.com@internal
+JR=$(H -o id rule add forward --in wan --to jfssh --service ssh --first --comment "e2e jarl")
+run "the set's addresses, CNAME followed" 0 H set jarl jfssh --refresh
+contains "ssh.example.com → jf.example.com  *internal  *$JF4  *yes"
+contains "ssh.example.com → jf.example.com  *internal  *$JF6  *yes"
+contains "net.example.com  *internal  *$NET6  *no: not within lan"
+run "in the ruleset" 0 R nft list set inet housecarl jfssh_v6
+contains "$JF6"
+not_contains "$NET6"
+run "the address outside is warned of" 0 H status
+contains "jarl_outside"
+run "IPv6 ssh to jellyfin through the Jarl set" 0 connect net $JF6 22
+contains "^ssh$"
+# The record changes in Jarl: within the refresh, the set follows.
+jarl_data 2001:db8:7f3d:10::99
+sleep 12
+run "the new address, without a change in Housecarl" 0 R nft list set inet housecarl jfssh_v6
+contains "2001:db8:7f3d:10::99"
+not_contains "$JF6"
+run "ssh to the old address is dropped again" 1 connect net $JF6 22
+run "no history entry for it" 0 H history list
+contains "update set jfssh"
+not_contains "10::99"
+run "an audit record" 0 H audit --action set.jarl
+contains "added 2001:db8:7f3d:10::99"
+# Jarl goes away: the set keeps what it had, over a restart too.
+jarl_data $JF6
+H set jarl jfssh --refresh >/dev/null
+R sh -c 'kill $(pidof fakejarl)'
+run "Jarl down: the set keeps its addresses" 0 H set jarl jfssh --refresh
+contains "Reading Jarl failed"
+contains "$JF6  *yes"
+run "still through" 0 connect net $JF6 22
+contains "^ssh$"
+run "warned" 0 H status
+contains "jarl_unreachable"
+stop
+run "restart housecarld, Jarl still down" 0 start
+run "the set is back from the database" 0 R nft list set inet housecarl jfssh_v6
+contains "$JF6"
+run "the boot ruleset has its addresses" 0 R grep -q "$JF6" /w/data/ruleset.nft
+run "still through after the restart" 0 connect net $JF6 22
+contains "^ssh$"
+run "the Jarl rule goes" 0 H rule delete -y "$JR"
+run "the Jarl set goes" 0 H set delete -y jfssh
+run "no Jarl warning left" 0 H status
+not_contains "jarl_"
+
 # ---- restart and boot
 
 stop

scripts/fakejarl/main.go

@@ -0,0 +1,63 @@
+// Command fakejarl answers the part of Jarl's API that Housecarl's Jarl
+// sets read (GET /zones, GET /zones/{id}/records), from a JSON file read
+// again on each request, so the e2e test can change records:
+//
+//	{"zones": [{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"}],
+//	 "records": {"1": [{"name": "nas.example.com.", "type": "A", "data": "10.0.0.5"}]}}
+//
+// Usage: fakejarl -listen 127.0.0.1:8053 -data jarl.json -token SECRET
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+)
+
+type data struct {
+	Zones   []map[string]any            `json:"zones"`
+	Records map[string][]map[string]any `json:"records"`
+}
+
+func main() {
+	listen := flag.String("listen", "127.0.0.1:8053", "address to listen on")
+	file := flag.String("data", "jarl.json", "the zones and records")
+	token := flag.String("token", "", "the API token Housecarl must send")
+	flag.Parse()
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer "+*token {
+			w.WriteHeader(http.StatusUnauthorized)
+			w.Write([]byte(`{"error": {"code": "unauthorized", "message": "bad token"}}`))
+			return
+		}
+		b, err := os.ReadFile(*file)
+		var d data
+		if err == nil {
+			err = json.Unmarshal(b, &d)
+		}
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		p := strings.TrimPrefix(r.URL.Path, "/api/v1")
+		switch {
+		case p == "/zones":
+			json.NewEncoder(w).Encode(d.Zones)
+		case strings.HasPrefix(p, "/zones/") && strings.HasSuffix(p, "/records"):
+			id := strings.TrimSuffix(strings.TrimPrefix(p, "/zones/"), "/records")
+			recs := d.Records[id]
+			if recs == nil {
+				recs = []map[string]any{}
+			}
+			json.NewEncoder(w).Encode(recs)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(`{"error": {"code": "not_found", "message": "not found"}}`))
+		}
+	})
+	log.Fatal(http.ListenAndServe(*listen, nil))
+}