Release 0.2.0-rc.4+git70-83b97ad

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitea/workflows/release.yml

@@ -0,0 +1,69 @@
+# Builds and publishes a release when a version tag (v0.2.0, v1.0.0-rc.1)
+# is pushed:
+#   * a Gitea release with the CHANGELOG notes, the linux tarballs, the CLI
+#     for Linux, macOS and Windows, the .debs and SHA256SUMS;
+#   * the .debs (amd64, arm64) in the owner's Debian package registry:
+#     distribution "stable", or "testing" for a pre-release (a version
+#     with a hyphen), which Gitea also marks as a pre-release.
+#
+# Needs a repository secret RELEASE_TOKEN: an access token of a user who
+# can write to the repository, with the write:repository and write:package
+# scopes. docs/development.md ("Releasing") has the steps.
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Check the tag against VERSION and CHANGELOG.md
+        run: |
+          version=$(cat VERSION)
+          if [ "$GITHUB_REF_NAME" != "v$version" ]; then
+            echo "::error::tag $GITHUB_REF_NAME does not match VERSION ($version)"
+            exit 1
+          fi
+          # The release notes (a pre-release may use its final version's
+          # section or Unreleased; see scripts/gitea-release.sh).
+          if ! scripts/gitea-release.sh notes > /dev/null; then
+            echo "::error::no release notes for $version in CHANGELOG.md"
+            exit 1
+          fi
+
+      - name: Tools
+        run: |
+          missing=
+          for t in jq dpkg-deb curl; do command -v $t >/dev/null || missing="$missing $t"; done
+          if [ -n "$missing" ]; then
+            sudo=; [ "$(id -u)" = 0 ] || sudo=sudo
+            $sudo apt-get update -qq && $sudo apt-get install -y -qq jq dpkg curl
+          fi
+
+      - name: Test
+        run: make test
+
+      - name: Build
+        env:
+          # Checked against VERSION above; don't depend on the checkout
+          # having the tag.
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          export DEB_MAINTAINER="$(git log -1 --format='%an <%ae>')"
+          make dist
+          make cli-dist
+          make deb DEB_ARCH=amd64
+          make deb DEB_ARCH=arm64
+
+      - name: Publish
+        env:
+          GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+        run: scripts/gitea-release.sh

AGENTS.md

@@ -1,8 +1,9 @@
 # Working on Housecarl
 
 Notes for coding agents (and people) changing this repository. Read this
-first, then [docs/design.md](docs/design.md), the plan and its open
-questions.
+first, then [docs/design.md](docs/design.md) (how it fits together and
+what 0.1 leaves out) and [docs/development.md](docs/development.md)
+(layout, tests, releasing).
 
 Housecarl is a firewall manager for Linux: `housecarld` keeps the
 machine's nftables ruleset and serves a REST API and a web UI; `hc` is the
@@ -12,9 +13,6 @@ their ways: the same auth, socket and web UI helpers, the same release
 process, the same look. When in doubt, do it the way Jarl and Steward do.
 Go module `housecarl`, `CGO_ENABLED=0`, Linux only for housecarld.
 
-Housecarl is at its start: only the version commands, the build and the
-brand exist. Most rules below are for what is coming.
-
 ## Ground rules
 
 * **The database is the source of truth.** The ruleset is rendered from
@@ -28,11 +26,21 @@ brand exist. Most rules below are for what is coming.
 * **Housecarl's own table only.** Write `inet housecarl` and nothing
   else; never flush the whole ruleset (Docker, libvirt and others keep
   theirs).
-* **Don't lock the user out.** Changes to the rules for the machine itself
-  are applied on probation and rolled back unless confirmed
-  (docs/design.md).
+* **Don't lock the user out.** Changes made over the network are applied
+  on probation and rolled back unless confirmed (docs/design.md). An
+  empty configuration applies nothing. But never force a rule that keeps
+  SSH or the web UI open: the maintainer doesn't want one.
+* **Shadow mode changes no packet** (docs/shadow.md): anything new the
+  renderer writes that drops, rejects, translates or alters packets needs
+  its shadow form (counted, marked `[would …]`) or must be left out
+  there. `TestShadow` fails on what slips through.
+* **Stay within nftables 1.1.3** (Debian 13). The dev machine's newer nft
+  accepts things 1.1.3 refuses; the tests with `HOUSECARL_NFT` and the
+  e2e test use 1.1.3.
 * **Every user-visible change updates `CHANGELOG.md`** under
   `## [Unreleased]`, and the matching `docs/` page in the same change.
+  New validation codes get a case in `internal/validate/validate_test.go`
+  and a row in `docs/validation.md`.
 * **Errors** say what is wrong and how to fix it, naming the object.
 * **Schema changes** append a migration; never edit a shipped one.
 * `make test` runs `go vet` and a `gofmt -l` check. `goimports` isn't
@@ -56,8 +64,25 @@ brand exist. Most rules below are for what is coming.
 
 ```sh
 make test      # vet + gofmt check + unit tests
+make e2e       # real traffic through a router container (needs Docker)
 ```
 
+* **Docker** on this machine needs `DOCKER_HOST=unix:///var/run/docker.sock`.
+  `docker build` can't reach the network here: images are made with
+  `docker run` + `docker commit` (scripts/e2e.sh makes `housecarl-nft`).
+  The e2e test prints `RESULT: N passed, 0 failed`; run it after any
+  change to `render`, `nft`, `engine`, `server`, `importer` or the CLI.
+* **Real nft in unit tests:**
+  `HOUSECARL_NFT="docker run --rm -i --cap-add NET_ADMIN housecarl-nft nft"`
+  and `HOUSECARL_NFT_IMAGE=housecarl-nft`; without them those tests skip.
+  Unprivileged namespaces (`unshare -rn`) are blocked here by AppArmor.
+* **The systemd sandbox is only tested by a package test**: `make deb`,
+  installed in a `debian:trixie` container running systemd
+  (`--privileged --cgroupns=host /sbin/init`).
+* **The web UI** has no JS tests. Bundle `assets/app.js` with esbuild to
+  catch syntax and import errors, and look at it in headless Chrome
+  against the e2e lab (`KEEP=1`, web UI on `10.88.10.1:8070`).
+
 ## Web UI rules
 
 As in Jarl and Steward:

CHANGELOG.md

@@ -6,14 +6,374 @@ All notable changes to Housecarl are documented here. The format follows
 
 ## [Unreleased]
 
-The start of Housecarl, a firewall manager for nftables, built the way
-Jarl and Steward are.
+### Added
+
+- **Sets from Jarl** ([docs/jarl.md](docs/jarl.md#sets-from-jarl)). A set
+  can hold the addresses of names in Jarl instead of listing them:
+  every A and AAAA record of each name, CNAMEs followed (within Jarl's
+  zones, in the same view), one view per name (`NAME@VIEW`; a name more
+  than one view covers must say which). housecarld reads them through
+  Jarl's API every `jarl.refresh` (a minute) and after each change, and
+  when they change applies the ruleset again, without a history entry
+  (audited as `set.jarl`). A set uses only the addresses `within` the
+  zones (their connected networks, which grow with the zone) and
+  networks it names; others are left out and warned of (`jarl_outside`).
+  While Jarl can't be read, sets keep the addresses they had, saved in
+  the database for restarts and in the boot ruleset (`jarl_unreachable`).
+  Names that give nothing say why (`jarl_name`: no record, a wildcard
+  only, a CNAME out of Jarl or in a loop, below a delegation, a secondary
+  zone, more than one view). `hc set add --jarl NAME@VIEW --within ZONE`,
+  `hc set jarl SET [--refresh]`; the kind "from Jarl" and Addresses… on
+  the Sets page; `GET /sets/{name}/jarl`, `POST /sets/{name}/jarl/refresh`.
+  Housecarl's Jarl token needs `records:read` for the zones.
+- An administrator can make an API token for another user in the web
+  UI: New token on the user's row under Users (the CLI's `hc token create
+  --user` and the API could already).
+- **Ban lists** ([docs/bans.md](docs/bans.md)). A program (Guard, which
+  reads logs, or fail2ban) bans an address, of every port or of some,
+  for a time; the ban list says what the address is kept from: the
+  firewall itself (`firewall`), some hosts behind it (`hosts`, port
+  forwards to them included), or `everything`. Bans are dropped ahead of
+  every rule, established connections too, and the banned address's
+  connections are closed. IPv6 addresses are banned by /64 (`group_v6`).
+  Never banned: inside networks, the firewall's own addresses, whoever
+  asks, and the list's `never`. Bans end by themselves, are cut to the
+  list's longest time, survive restarts and reboots, and are audited, but
+  aren't configuration (no history, probation or changesets). `hc
+  banlist`, `hc bans`, `hc ban`, `hc unban`; Traffic → Bans in the web UI;
+  `/banlists` and `/banlists/{ref}/bans` in the API; a `bans` grant that
+  can name lists (`bans:write:web`), in the `viewer` and `operator`
+  roles.
+- **A fail2ban action**, `contrib/fail2ban/action.d/housecarl.conf`
+  (installed under `/usr/share/doc/housecarl/contrib/`): fail2ban's bans
+  go into a ban list, with the jail's port and protocol.
+- **Failed logins in the journal.** A failed login and a refused API
+  token are logged (`msg="login failed" ip=… user=…`), for Guard and
+  fail2ban to act on.
+
+- **Quota usage as a share.** What a quota used shows its percentage
+  (`12.3 GB of 50.0 GB used (24%)`, each address's in the quota
+  dialog), on the Rules page, the dashboard and in `hc quotas`.
+- **Reset one address's quota, or give some back.** `hc rule
+  reset-quota ID --address ADDR` starts only that device's allowance
+  again; `--give 5gbytes` takes that much off what was used instead (one
+  address's, or everyone's): extra allowance until the next reset,
+  without changing the rule. In the web UI, a rule's quota is one
+  line (what it used, how it resets) that opens a dialog with the rest:
+  usage per address, last and next reset, and Reset and Give back for
+  everyone or one address. `POST /rules/{ref}/reset-quota` takes `address` and `give`.
+- **Names beside inside addresses.** Connections, the log, quota usage,
+  the dashboard's quotas and the statistics' devices and blocked
+  sources show each inside address's name, found by a reverse lookup:
+  `laptop 192.168.1.20` in the web UI, `192.168.1.20 (laptop)` in `hc`.
+  Only inside addresses are looked up (an inside zone's networks, and
+  private ones outside the external zones'). Names are cached for 10
+  minutes and never stored. `names.enabled`, `names.resolver` and
+  `names.cache` in housecarl.yaml; `hc --no-names`; `POST /names`.
+- **Quotas on the dashboard.** With any rule holding a quota, the
+  dashboard lists them all: the allowance, what is used (for a
+  per-address quota, how many addresses and the busiest) with a bar, and
+  the next reset. Each opens its rule.
+- **Links to a rule open it.** A rule named in the matrix, the log, the
+  tracer, statistics or the dashboard's busiest rules opens the Rules
+  page on its chain, scrolled to it and highlighted (`#/rules?rule=ID`).
+  The Rules page shows each rule's ID (`#410`, as `hc` and labels name
+  it) beside its order.
+- **A clearer zone matrix.** A cell reads allow, block, or **partly**
+  (some new connections let through, the rest not), with the exceptions
+  counted by what they do (`+3 -1` instead of `+4`), and what they let
+  through: services with the addresses they're limited to (`ssh from
+  trusted`, `5 port forwards`), under the grid in `hc matrix`, in each
+  cell on the Matrix page. A zone to itself is marked (only traffic routed
+  between its interfaces passes the firewall). The details list each
+  rule's hits. API: `summary`, `allowed`, `blocked`, `open`,
+  `same_zone`, `note`, `hits`, and `short` and `hits` per exception.
+  Raw rules that are a match and then accept, drop or reject (without
+  interfaces) are read instead of shown as unknown.
+- **Shadow mode** (`shadow.enabled` in housecarl.yaml; docs/shadow.md):
+  Housecarl's ruleset is loaded beside the firewall you have but blocks,
+  rejects and translates nothing. Drops and rejects are counted and accept
+  instead (an accept only ends Housecarl's chain), policies accept and
+  always log what they would drop, NAT only counts, MSS clamping and the
+  flowtable are left out, and raw rules that do more than accept, drop or
+  reject are left out with a warning (`shadow_raw_left_out`). Comments
+  mark what would happen (`[would drop]`). `shadow.position`: Housecarl's
+  filter chains run after the other firewall's (`filter + 10`, the
+  default: what it would drop is what switching would break) or before
+  (`- 10`); its NAT chains always first. Every change is checked by
+  nftables in both forms, so leaving shadow mode can't fail. No
+  probation; takeovers and closing connections are refused (409). The
+  boot ruleset is the shadow one, and its fallback filters nothing. When
+  a reload (`flush ruleset`) deletes Housecarl's table, housecarld puts it
+  back (it listens to nftables' netlink notifications). `mode` in
+  `/status`, `/healthz` and `/auth/whoami`; a banner in the web UI;
+  would-be drops marked in the log (`shadow`) and the tracer
+  (`would_drop`); `hc ruleset --enforcing` (`GET /ruleset?enforcing=1`)
+  shows the ruleset for after. `shadow_unprotected` warns when no other
+  table filters the input hook; `nftables_service` and `takeover_pending`
+  aren't raised in shadow mode. The e2e test imports in shadow mode
+  before taking over.
+- **Zone names at import:** `hc import --zone enp2s0=lan` (repeat;
+  interfaces given one name share a zone; Zone names on the Import
+  page), and a note listing each zone and where its name came from.
+  Defines holding a VLAN sub-interface (`enp2s0.20`) name their zone,
+  and `_ifname`, `_device` and `_nic` are taken off define names too
+  (docs/import.md, "How things are named").
+- Import: sets with `flags timeout` become dynamic sets; a DNAT of
+  several ports becomes one port forward per port.
+
+### Changed
+
+- `hc user delete` asks first, as every other delete does, and takes
+  `-y`/`--yes` not to; it deleted the user and their tokens at once.
+- The ruleset's diff (Changes, `hc changeset diff`, `--dry-run`) names
+  the chain each part is in on its `@@` line (`@@ -40,6 +40,8 @@ chain
+  forward`), as `git diff` names the function, so a rule added deep in a
+  long chain says whether it is in input, forward or output.
+- **History shows the ruleset's diff.** An entry's page (and `hc history
+  diff ID`, `GET /history/{id}/diff`) shows what the change did to the
+  ruleset, for entries made before this version too: the configuration
+  before and after it is got back from the current one and rendered by
+  this version.
+- Negating a match is said first: the zone, From and To fields (rules
+  and NAT) start with "any of" / "any but" instead of a checkbox under
+  the list, shown once the list has an entry, and an "any but" list is
+  marked with a red bar. An empty list is never saved as negated (it
+  matched everything anyway, but would have negated what was added to
+  it later).
+- docs/rules.md: "Matching everything but a few", on leaving some
+  addresses out with a zone and "any but", and what to do when the range
+  isn't a zone.
+- The rule dialog shows only the zone fields its chain can use: no
+  "Going out on" for input and early, no "Coming in on" for output
+  (they could only be refused). A zone left in a field the chain hides
+  isn't sent.
+- History lists a changed setting by its own fields
+  (`builtins.drop_invalid: false → true`), on the web page and in `hc
+  history show`, instead of the whole `builtins` before and after.
+- Set names starting with `ban_` are now reserved (`set_name_reserved`):
+  ban lists' sets use them.
+- **Rules read as a direction.** A match's zones are written `lan → wan`,
+  `any but wan → any`, or with the firewall itself as the other end
+  (`lan → firewall` in input, `firewall → wan` in output), instead of
+  `in lan · out wan` and `in not wan`; exclusions read `from any but X`.
+  On the Rules and NAT pages and in `hc`; the flags and JSON stay as
+  they were.
+
+### Fixed
+
+- An IPv4-mapped prefix (`::ffff:0:0/96`) is refused with an error that
+  says so; it was refused as "has host bits set; the prefix is invalid
+  Prefix".
+- `hc forward show` shows the forward's hits and comment, and its
+  hairpin as `no`, `yes` or `yes, from …` (it printed `no -`).
+- A history entry's ruleset diff showed the object it changed as moved:
+  stepping back put the old form at the end of its list, so updating
+  one port forward (turning hairpin off) also showed its forward and
+  DNAT lines moving down. The diff is now only what the change did, and
+  its sides are named `before N` and `after N` instead of `now`.
+- Ruleset diffs numbered the hunks after one with a moved line too early
+  on the new side (`@@ -76,9 +74,9 @@` for `+76,9`), so the chain named
+  after `@@` could be the one before (`chain output` for a hunk in
+  `chain dstnat`). go-udiff miscounts them; Housecarl now numbers them
+  itself.
+- The dashboard's bar for a per-address quota showed only the busiest
+  address, so one address that used up its allowance filled it (100%).
+  The bar is now all the addresses together, and the dashboard and the
+  rule's quota line say how many addresses used theirs up.
+- Changing how a quota counts lost or muddled what it used: per address
+  to a total started at nothing, per source to per destination kept the
+  sources' use as if it were the destinations', and other groups left
+  the old entries behind. Now a total starts at the addresses' sum,
+  wider groups add up their addresses, and what can't be carried starts
+  again with a warning (`quota_restarts`) on the change.
+- What the quotas used is saved after every change, not only every five
+  minutes: a reboot soon after a change that counts a quota differently
+  could bring back its usage counted the old way.
+- Statistics' Quotas card linked every entry to the Rules page, not to
+  its rule; each now opens its rule, named. Per-address quotas were
+  missing from it (and from the `quota:rN` series): they count their
+  addresses' sum.
+- `hc stats top devices` listed each device twice (once for what it
+  received, once for what it sent); it is one row with both again.
+- `hc rule add --before/--after/--first` made two changes (create at the
+  end, then move), so two history entries, two probations, and the rule
+  briefly in force at the end of its chain; staged in a changeset, the
+  place was lost. The create now takes the place (`POST
+  /rules?before=ID`, `after`, `first`) and makes one change.
+- `hc matrix` in a terminal: the colours' escape codes threw the columns
+  out of line.
+- `hc counters` (`GET /counters`) kept listing the objects of a
+  configuration replaced since housecarld started (`hc import
+  --replace` gives every object a new ID), beside the current ones; the
+  totals of objects that no longer exist are now forgotten.
+- Import: two base chains on one hook had their rules imported twice,
+  and the second chain's leading built-ins were moved above the first
+  chain's rules; they now follow each other by priority (read in a
+  fixed order), with the stricter policy, and a note.
+- Import: a rule inside a jumped-to chain that stays raw kept only its
+  own text and so applied to everything; the jump's condition now goes
+  in front. A jump and a rule matching the same field are combined only
+  when one's addresses lie within the other's; else the rule stays raw.
+  A jump's ports, ICMP types, connection status and hop limit were
+  dropped; a limit or log on the jump is no longer silently lost.
+- Import: a second match on the same field (`ip saddr A ip saddr != B`)
+  replaced the first; `!=` on the hop limit and TCP flags was read as
+  `==`. Such rules now stay raw, as written.
+- Import: a forward-chain accept to a forwarded host and port without
+  `ct status dnat` was folded into the port forward, which only lets
+  forwarded connections through; it stays a rule now. An IPv6 accept
+  limited to an interface no longer becomes a pinhole open to all.
+- Import: raw rules naming a set of the old table made nftables refuse
+  the ruleset; address sets are now renamed `NAME_v4`/`NAME_v6`, other
+  sets written out with their elements, and what can't be is left out
+  with a note. Rules naming a set that isn't imported stay raw instead
+  of failing validation.
+- Import: a DNAT to an IPv6 address, or with matches a port forward
+  can't hold (an interface excluded, a limit, a log), is left out with a
+  note instead of being imported without them.
+
+## [0.1.0] - 2026-10-19
+
+The first version of Housecarl, a firewall manager for nftables, built
+the way Jarl and Steward are.
 
 ### Added
 
+- **Zones, address sets and services:** interfaces grouped by what they
+  connect to (wildcards like `incusbr*`; external zones face the
+  internet); sets of IPv4 and IPv6 addresses, prefixes and ranges, mixed;
+  ports and ICMP types by name, with twenty built in (docs/rules.md).
+- **Rules** in four chains (input, forward, output, and early, before
+  connection tracking), in order, matching by zone, address, service,
+  family, connection state and status, days and hours, a rate (also per
+  source address), and a connection count; accept, drop, reject (ICMP or
+  TCP reset) or log. Raw nftables rules for the rest.
+- **IPv4 and IPv6 together:** one rule becomes the nftables rules each
+  family needs.
+- **NAT:** port forwards as one object (DNAT, the forward accept, hairpin
+  NAT, and optionally an IPv6 pinhole to a host's own port); source NAT
+  (masquerade or an address; NAT66 on request); redirects to the firewall
+  (docs/nat.md).
+- **Built-ins**, each switchable: loopback, established connections,
+  invalid packets, ICMPv6 (RFC 4890), ping from internal zones, DHCPv6
+  client replies, antispoofing with your own prefixes, MSS clamping, and
+  a flowtable fast path.
+- **Changesets, as in Jarl:** changes staged to be reviewed and applied
+  together, as one history entry. Each is checked as it is staged; the
+  review shows the problems, nftables' check and the diff of the ruleset.
+  The web UI collects edits as pending changes by default (pending rows,
+  a bar with Review and Apply now, a Changes page), with a switch for live
+  edits, and Undo in History is staged like any edit; hc stages with
+  `-C ID` or `HOUSECARL_CHANGESET`, and has `hc
+  changeset create/list/show/diff/apply/discard/remove`. An applied
+  changeset is on probation like any change. `--dry-run` shows the
+  ruleset's diff too (docs/design.md, docs/cli.md).
+- **Bandwidth limits and quotas:** rate limits in bytes (`--limit
+  20mbytes/second,over,per-source`), per source or per destination;
+  quotas, an allowance of traffic per rule or per address, reset daily,
+  weekly, monthly on a chosen day, or by hand, kept across changes,
+  restarts and reboots (`hc quotas`, `hc rule reset-quota`, usage on the
+  rules page). Rules that count traffic are checked before established
+  connections are accepted, so they see every packet. Per-address limits
+  group IPv6 by /64 by default (`--group-v4`, `--group-v6`). Meters,
+  byte rates and quotas in an imported ruleset carry over
+  (docs/rules.md).
+- **Changes checked twice and applied atomically:** validation of the
+  whole configuration with stable problem codes (docs/validation.md),
+  then `nft -c`, then one `nft -f` transaction that replaces Housecarl's
+  own table, `inet housecarl`, and nothing else. Dynamic sets' entries
+  and the counters carry across.
+- **Probation:** a change made over the network rolls itself back unless
+  confirmed within two minutes. The web UI and `hc` confirm their own
+  changes once they reach Housecarl again over a new connection, from the
+  same address (a change that cuts its client off can't, and rolls back);
+  `--no-confirm` and the "a person confirms each change" setting leave it
+  to `hc confirm` or the banner;
+  changes through the local socket are kept at once; a restart rolls
+  back what was pending. No rule is forced to keep SSH or the web UI
+  open.
+- **At boot:** the confirmed ruleset is loaded by
+  `housecarl-ruleset.service` before the network comes up, without
+  housecarld. An empty configuration applies nothing. It fails closed:
+  if the saved ruleset doesn't load, a fallback saved beside it
+  (`ruleset-fallback.nft`) is, until housecarld puts the real one back:
+  nothing new gets in from external zones, the inside keeps working
+  (masqueraded), and the unit is marked failed with a critical message.
+  `hc status`, the dashboard and `/healthz` say while the fallback is in
+  force; the warning `no_external_zone` says when no zone is external,
+  which would make the fallback close everything (docs/operations.md).
+- **The flowtable and interfaces that come and go:** the boot ruleset
+  has no flowtable (nftables on Debian 13's kernel refuses one naming a
+  device that doesn't exist yet, such as `ppp0` before PPPoE is up, and
+  with it the whole file); housecarld watches the kernel's link messages
+  and applies the ruleset again when a flowtable device appears or comes
+  back as a new interface (PPPoE reconnecting). A guide to when the
+  flowtable helps, and what it costs (docs/flowtable.md).
+- **`GET /healthz` for monitoring:** `ok`, `firewall` (applied,
+  unconfigured, fallback, missing), changes on probation and the
+  machine's warnings, with HTTP 503 unless the firewall is in force;
+  no login needed, read at most every five seconds.
+- **Import keeps a chain's head in order:** once a would-be built-in at
+  the head of a chain has to stay a rule (an invalid drop the forward
+  chain needs after an exception), the built-ins after it stay rules
+  too, in their place, instead of moving ahead of it: an ICMPv6 accept
+  no longer jumps ahead of the invalid drop it came after.
+- **Renaming a zone, set or service** changes the rules, NAT rules and
+  port forwards that name it, in the same change (one history entry, one
+  undo); before, a name in use couldn't change. Dynamic sets keep their
+  names (their tokens and programs go by them).
+- VLAN sub-interfaces (`enp2s0.20`) in zones and the flowtable, tested
+  with tagged traffic (docs/rules.md).
+- `hc settings set --flowtable none` (and `none` for the other lists)
+  empties the list.
+- **Warnings about the machine:** while `nftables.service` is enabled
+  (it loads `/etc/nftables.conf` at boot, and flushes every table when
+  stopped), and while the table an import came from still applies.
+- **Statistics** (Traffic → Statistics, `hc stats`, `/api/v1/stats/...`):
+  traffic per interface and zone, drops by chain and when they happen,
+  hits per rule and the rules that matched nothing, the sources blocked
+  most and what they tried, connections against the kernel's limit, each
+  quota's use, and housecarld's and the host's memory and load, sampled
+  every minute into a database of their own (`stats.db`, kept 7 days per
+  minute, 90 days per hour, 2 years per day). On by default; traffic per
+  inside device is a setting (`--stats-per-device`) that adds counters
+  to the forward chain. Two charts on the dashboard; a warning when the
+  connection tracking table is nearly full (docs/stats.md).
+- Set names starting with `meter_`, `conn_`, `quota_` or `stats_` are
+  refused (`set_name_reserved`): Housecarl's own sets use them, and a set
+  of yours named so could clash with them.
+- **The zone matrix** (Matrix, `hc matrix`): what the rules do to new
+  connections between zones and to and from the firewall, worked out
+  from the rules in order, with each pair's exceptions; rules about
+  networks connected to other zones are left out of a pair.
+- **Connections and tracing:** the connections the kernel tracks, with
+  their NAT, and closing them so the rules check them again (Traffic →
+  Connections, `hc connections`); a rule option, "every packet", for
+  blocks that stop connections already open; and a packet tracer
+  (Traffic → Trace, `hc trace`) that follows packets through every
+  table's rules and names what accepted or dropped them.
+- **Hits and the log:** packet and byte counters for every rule, NAT rule,
+  built-in and policy; packets logged by rules and policies, read from
+  netlink log group 4171 (`hc log -f`, the Log page).
+- **Dynamic sets** whose entries expire by themselves, changed through
+  their own API with tokens that may be limited to one set
+  (`sets:write:guests`).
+- **Import** of an existing nftables ruleset (`hc import`): rules keep
+  their order, defines name the sets, DNAT becomes port forwards, what
+  can't be translated stays as raw rules, with notes; `hc takeover`
+  deletes the old table, on probation (docs/import.md). DNAT maps and
+  verdict maps are expanded into one port forward or rule per element.
+- **The `hc` CLI, the REST API and the web UI** (dashboard, rules per
+  chain with hits and reordering, NAT, zones, sets, services, the log,
+  settings, import, history with undo; the rule dialog keeps the less
+  used parts, such as rate limits, quotas and time windows, behind "+"
+  buttons; each chain's built-ins are listed, with their hits, and
+  switched in a window), with users, roles, scoped API
+  tokens, sessions, the audit log, and the web UI's certificate from
+  Let's Encrypt through Jarl, as in Steward.
+- **Packaging:** Debian packages `housecarl` and `housecarl-cli`, a
+  systemd unit whose only privilege is `CAP_NET_ADMIN`, daily backups.
 - **The Housecarl logo:** a carved Nordic doorway in Jarl's blue, with a
   simplified version for small sizes, lighter on dark backgrounds; and a
   HOUSECARL wordmark in Jarl's and Steward's angular letters.
-- **The skeleton:** the `housecarld` daemon and the `hc` CLI (both only
-  print their version), the build, and the design sketch
-  (docs/design.md).

Makefile

@@ -3,6 +3,12 @@
 #   make              build bin/housecarld and bin/hc (and their bash completion)
 #   make test         vet, gofmt check and unit tests
 #   make logo         rebuild the web UI's logo and the README banner from artwork/
+#   make e2e          end-to-end test with real traffic through a firewall in Docker (scripts/e2e.sh)
+#   sudo make install install binaries, bash completion, systemd units,
+#                     sysusers/tmpfiles files and an example /etc/housecarl/housecarl.yaml
+#   make deb          Debian packages in dist/ (DEB_ARCH=arm64 for another
+#                     architecture): housecarl (the server) and housecarl-cli (hc,
+#                     which housecarl depends on)
 #   make dist         static binaries for linux/amd64 and linux/arm64 in dist/
 #   make cli-dist     the CLI alone for Linux, macOS and Windows in dist/
 #                     (housecarl-cli-<version>-<os>-<arch>[.exe])
@@ -23,7 +29,32 @@ LDFLAGS := -s -w -X housecarl/internal/version.Version=$(VERSION) -X housecarl/i
 GOFLAGS := -trimpath
 export CGO_ENABLED := 0
 
-.PHONY: all build completion test vet logo dist cli-dist clean
+PREFIX      ?= /usr/local
+BINDIR      ?= $(PREFIX)/bin
+DOCDIR      ?= $(PREFIX)/share/doc/housecarl
+# bash-completion also looks in /usr/local/share (XDG_DATA_DIRS).
+BASHCOMPDIR ?= $(PREFIX)/share/bash-completion/completions
+UNITDIR     ?= /etc/systemd/system
+SYSUSERSDIR ?= /usr/lib/sysusers.d
+TMPFILESDIR ?= /usr/lib/tmpfiles.d
+DESTDIR     ?=
+
+# The binaries install-files copies (make deb points these at other
+# architectures' builds).
+HOUSECARLD_BIN ?= bin/housecarld
+HC_BIN         ?= bin/hc
+
+# make deb: the package's architecture (Debian name) and version. A
+# release is VERSION with ~ for - (0.1.0-rc.1 -> 0.1.0~rc.1, which apt
+# sorts before 0.1.0). A development build is versioned after the last
+# tag (0.1.0~rc.1+git3.gca0f148); uncommitted changes add the build time.
+DEB_ARCH    ?= $(shell dpkg --print-architecture 2>/dev/null || go env GOARCH)
+DEB_DEV_VERSION := $(shell [ -n '$(DESCRIBE)' ] || exit 0; \
+	m=; git diff --quiet HEAD 2>/dev/null || m=.m$$(date -u +%Y%m%d%H%M%S); \
+	echo '$(DESCRIBE)' | sed -E 's/^v(.*)-([0-9]+)-g([0-9a-f]{7})[0-9a-f]*$$/\1+git\2.g\3/' | tr - '~' | sed "s/$$/$$m/")
+DEB_VERSION ?= $(if $(RELEASE_TAG),$(subst -,~,$(VERSION)),$(or $(DEB_DEV_VERSION),$(subst -,~,$(VERSION))))
+
+.PHONY: all build completion test vet logo e2e install install-files uninstall deb dist cli-dist clean
 
 all: build
 
@@ -48,6 +79,56 @@ test: vet
 logo:
 	python3 scripts/logo.py
 
+e2e:
+	scripts/e2e.sh
+
+install: build install-files
+	@if [ ! -e $(DESTDIR)/etc/housecarl/housecarl.yaml ]; then \
+		install -d -m 0750 $(DESTDIR)/etc/housecarl; \
+		bin/housecarld config --example > $(DESTDIR)/etc/housecarl/housecarl.yaml; \
+		chmod 0640 $(DESTDIR)/etc/housecarl/housecarl.yaml; \
+		echo "wrote $(DESTDIR)/etc/housecarl/housecarl.yaml"; \
+	fi
+	@echo
+	@echo "Next (as root): systemd-sysusers housecarl.conf && systemd-tmpfiles --create housecarl.conf"
+	@echo "                chgrp housecarl /etc/housecarl /etc/housecarl/housecarl.yaml && systemctl daemon-reload"
+	@echo "                systemctl enable housecarl-ruleset.service"
+	@echo "Then follow docs/install.md (init, import your ruleset, start)."
+
+# Everything but /etc/housecarl/housecarl.yaml (shared by install and deb).
+install-files:
+	install -D -m 0755 $(HOUSECARLD_BIN) $(DESTDIR)$(BINDIR)/housecarld
+	install -D -m 0755 $(HC_BIN) $(DESTDIR)$(BINDIR)/hc
+	install -D -m 0644 bin/completion/hc $(DESTDIR)$(BASHCOMPDIR)/hc
+	install -D -m 0644 bin/completion/housecarld $(DESTDIR)$(BASHCOMPDIR)/housecarld
+	for u in housecarl.service housecarl-ruleset.service housecarl-backup.service housecarl-backup.timer; do \
+		install -D -m 0644 deploy/systemd/$$u $(DESTDIR)$(UNITDIR)/$$u && \
+		sed -i 's#/usr/local/bin/housecarld#$(BINDIR)/housecarld#; s#/usr/local/share/doc/housecarl#$(DOCDIR)#' $(DESTDIR)$(UNITDIR)/$$u || exit 1; \
+	done
+	install -D -m 0644 deploy/sysusers.d/housecarl.conf $(DESTDIR)$(SYSUSERSDIR)/housecarl.conf
+	install -D -m 0644 deploy/tmpfiles.d/housecarl.conf $(DESTDIR)$(TMPFILESDIR)/housecarl.conf
+	install -D -m 0644 README.md $(DESTDIR)$(DOCDIR)/README.md
+	install -D -m 0644 LICENSE $(DESTDIR)$(DOCDIR)/LICENSE
+	install -D -m 0644 CHANGELOG.md $(DESTDIR)$(DOCDIR)/CHANGELOG.md
+	install -d $(DESTDIR)$(DOCDIR)/docs
+	install -m 0644 docs/*.md $(DESTDIR)$(DOCDIR)/docs/
+	install -D -m 0644 contrib/fail2ban/action.d/housecarl.conf $(DESTDIR)$(DOCDIR)/contrib/fail2ban/action.d/housecarl.conf
+
+uninstall:
+	rm -f $(DESTDIR)$(BINDIR)/housecarld $(DESTDIR)$(BINDIR)/hc $(DESTDIR)$(UNITDIR)/housecarl.service \
+		$(DESTDIR)$(UNITDIR)/housecarl-ruleset.service \
+		$(DESTDIR)$(UNITDIR)/housecarl-backup.service $(DESTDIR)$(UNITDIR)/housecarl-backup.timer \
+		$(DESTDIR)$(BASHCOMPDIR)/hc $(DESTDIR)$(BASHCOMPDIR)/housecarld \
+		$(DESTDIR)$(SYSUSERSDIR)/housecarl.conf $(DESTDIR)$(TMPFILESDIR)/housecarl.conf
+	rm -rf $(DESTDIR)$(DOCDIR)
+	@echo "Kept /etc/housecarl, /var/lib/housecarl and /var/backups/housecarl. The ruleset in force stays until the next boot."
+
+# Debian packages, built with dpkg-deb (no root needed); scripts/mkdeb.sh
+# explains the layout.
+deb: build
+	DEB_ARCH='$(DEB_ARCH)' DEB_VERSION='$(DEB_VERSION)' VERSION='$(VERSION)' \
+		LDFLAGS='$(LDFLAGS)' GOFLAGS='$(GOFLAGS)' scripts/mkdeb.sh
+
 dist: build
 	for arch in amd64 arm64; do \
 		GOOS=linux GOARCH=$$arch go build $(GOFLAGS) -ldflags '$(LDFLAGS)' -o dist/housecarl-$(VERSION)-linux-$$arch/housecarld ./cmd/housecarld && \

README.md

@@ -4,7 +4,7 @@
 
 *Every Jarl has a housecarl at his side; this one keeps the door.*
 
-Housecarl will be a firewall for Linux that you manage through a web UI, a
+Housecarl is a firewall for Linux that you manage through a web UI, a
 REST API and a command line client (`hc`), all three backed by the same
 API. It is the third of a set with [Jarl](https://gitea.adbraun.lol/adam/jarl),
 the BIND manager, and [Steward](https://gitea.adbraun.lol/adam/steward), the
@@ -17,24 +17,126 @@ door and decides who comes in. Housecarl does that for your network. It
 keeps the machine's nftables ruleset, built from its own database, in one
 table of its own.
 
-## What it will do
+## What it does
 
-* **Zones** for your interfaces and VLANs (`lan`, `iot`, `guest`, `wan`),
-  with a default policy between each pair, and rules between them.
-* **Address sets** that rules refer to, whose entries can expire by
-  themselves; another program can be allowed to change just one set. That
-  is how Steward's captive portal will let a guest in.
-* **NAT**: masquerading and port forwards.
-* **Changes you can trust**: validated as a whole, checked by nftables
-  before they are applied, applied in one transaction, and undoable. A
-  change that could lock you out is rolled back unless you confirm it.
+* **Zones, sets and services, then rules.** Name your interfaces (`wan`,
+  `lan`, `wg`), your addresses (`trusted`, IPv4 and IPv6 in one set) and
+  your ports (`web`, or the built-in `ssh`, `dns`, `https`…), and
+  write ordered rules with them: in, out or through the firewall, by
+  zone, address, service, connection state, time of day, rate per
+  source, or connection count. Anything else can be written as an
+  nftables rule by hand.
+* **Bandwidth and quotas.** Byte-rate limits per device, and traffic
+  allowances that reset daily, weekly or monthly (or by hand), counted
+  across changes and reboots.
+* **IPv4 and IPv6 together.** One rule covers both; Housecarl writes the
+  nftables rules each family needs.
+* **NAT in one object.** A port forward is the DNAT, the forward accept
+  and the hairpin NAT together, and can open the same port to an IPv6
+  host. Masquerading, SNAT and redirects too.
+* **The rules every firewall needs**, built in and each one switchable:
+  loopback, established connections, invalid packets, the ICMPv6 that
+  IPv6 needs, antispoofing for your prefixes, MSS clamping for PPPoE, the
+  DHCPv6 client, and a flowtable fast path
+  ([docs/flowtable.md](docs/flowtable.md)).
+* **Changes you can trust.** Each change is validated as a whole, checked
+  by nftables itself, and applied in one transaction. As in Jarl, edits
+  can be collected in a changeset, reviewed with the diff of the ruleset,
+  and applied (or discarded) together. A change made over
+  the network is on probation: unless you confirm it within two minutes,
+  it rolls itself back, so a rule that cuts you off undoes itself.
+* **Your existing ruleset, imported.** `hc import` turns an nftables
+  ruleset into zones, sets, services, rules and port forwards, keeping the
+  rules' order, and keeps what it can't translate as raw rules
+  ([docs/import.md](docs/import.md)).
+* **Try it first, in shadow mode.** Housecarl's ruleset runs beside the
+  firewall you have, blocking nothing: its hits, log and tracer show what
+  it would drop, so you see what switching over would break before you do
+  ([docs/shadow.md](docs/shadow.md)).
+* **Ban lists for Guard and fail2ban.** A program on any machine bans an
+  address it caught misbehaving in its logs, and the ban list says what
+  that address is kept from: the firewall, some hosts, or everything
+  ([docs/bans.md](docs/bans.md)).
+* **Easy to watch.** A zone matrix of what is allowed between zones,
+  statistics over time (traffic per zone and device, drops, the sources
+  blocked most, connections; [docs/stats.md](docs/stats.md)), hits per rule, a live log of what was dropped and
+  why, the connections the firewall tracks (and closing them), a packet
+  tracer that names the rule that dropped something, in any table, and
+  the history of every change, field by field, with undo.
+* **Its own table only.** Docker, Incus and libvirt keep theirs. The
+  confirmed ruleset is loaded at boot, before the network comes up, even
+  if housecarld doesn't start; should it fail to load, a fallback keeps
+  anything new from the internet out until housecarld puts it back.
+* **Sets other programs can fill.** A dynamic set's entries expire by
+  themselves, and a token can be limited to one set: that is how
+  Steward's captive portal will let a guest in.
+* **Users, tokens and HTTPS** as in Jarl and Steward, with the web UI's
+  certificate from Let's Encrypt through Jarl
+  ([docs/certificates.md](docs/certificates.md)).
 
-See [docs/design.md](docs/design.md) for the plan and the open questions.
+Not yet: rules by MAC address or DNS name, and iptables import
+([docs/design.md](docs/design.md)).
+
+## Quick start
+
+On Debian 13, from the package registry (see
+[docs/install.md](docs/install.md) for other ways):
+
+```sh
+sudo apt install housecarl
+sudo -u housecarl housecarld init          # asks for the admin password
+sudo systemctl enable --now housecarl      # applies nothing until configured
+
+# Import the ruleset you have...
+sudo hc import --defines /etc/nftables.conf -n      # look first
+sudo hc import --defines /etc/nftables.conf
+sudo hc takeover "inet main"
+sudo systemctl disable nftables.service
+
+# ...or start from scratch: let yourself in first
+sudo hc set add admins 192.168.1.0/24
+sudo hc rule add input --from admins --service ssh
+sudo hc zone add wan --interface ppp0 --external
+sudo hc zone add lan --interface eth1
+sudo hc rule add forward --in lan --out wan
+sudo hc snat add --out wan
+sudo hc status
+```
+
+The web UI is on `http://127.0.0.1:8070/` (see `listen` in
+`/etc/housecarl/housecarl.yaml`); write a rule that lets you reach it.
+
+## Documentation
+
+* [install.md](docs/install.md): packages, building from source, first steps
+* [rules.md](docs/rules.md): zones, address sets, services, rules and the built-ins
+* [nat.md](docs/nat.md): port forwards, hairpin NAT, IPv6 pinholes, source NAT, redirects
+* [bans.md](docs/bans.md): ban lists, where Guard and fail2ban ban addresses
+* [import.md](docs/import.md): moving from an existing nftables ruleset
+* [shadow.md](docs/shadow.md): shadow mode, Housecarl beside your firewall, changing nothing
+* [operations.md](docs/operations.md): probation, the boot and its fallback, hits and the log, monitoring, backups, troubleshooting
+* [flowtable.md](docs/flowtable.md): the fast path for forwarded traffic, and when (not) to use it
+* [certificates.md](docs/certificates.md): HTTPS with a certificate from Let's Encrypt, through Jarl
+* [jarl.md](docs/jarl.md): connecting Housecarl to Jarl, and what its token may do
+* [cli.md](docs/cli.md): the `hc` command line client
+* [api.md](docs/api.md): the REST API
+* [configuration.md](docs/configuration.md): `/etc/housecarl/housecarl.yaml`
+* [permissions.md](docs/permissions.md): users, roles, grants and API tokens
+* [validation.md](docs/validation.md): every check and what it means
+* [design.md](docs/design.md): how Housecarl is built, and what 0.1 leaves out
+* [development.md](docs/development.md): building, testing, releasing
 
 ## Status
 
-Housecarl is at its start: it has its name, its logo and the skeleton of
-the daemon (`housecarld`) and CLI (`hc`). There is nothing to run yet.
+Housecarl 0.2.0 is in testing (0.2.0-rc.4+git70-83b97ad): zones, address sets and
+services with ordered rules, IPv4 and IPv6, port forwards with hairpin NAT
+and IPv6 pinholes, source NAT and redirects, built-ins, probation for
+network changes (confirmed by the client that made them), the ruleset
+at boot with a fail-closed fallback, hits and the drop log, nftables
+import, changesets reviewed with the ruleset's diff (as in Jarl),
+bandwidth limits and quotas, shadow mode beside another firewall, ban
+lists for Guard and fail2ban, sets of names from Jarl, a web UI, REST
+API and CLI.
 
 ## Licence
 

VERSION

@@ -1 +1 @@
-0.1.0-dev
+0.2.0-rc.4+git70-83b97ad

cmd/hc/admin.go

@@ -0,0 +1,675 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+func historyCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "history", Short: "The history of configuration changes, and undo"}
+	var limit int
+	list := &cobra.Command{
+		Use: "list", Short: "List recent changes", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var hs []model.HistoryEntry
+			if err := c.Get(ctx(), "/history", q("limit", strconv.Itoa(limit)), &hs); err != nil {
+				return err
+			}
+			return output(hs, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tWHEN\tBY\tVIA\tCHANGES\tDESCRIPTION")
+				for _, h := range hs {
+					desc := h.Description
+					if h.RevertedBy != 0 {
+						desc += fmt.Sprintf(" (undone by %d)", h.RevertedBy)
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\t%s\n", h.ID, h.Time.Local().Format("2006-01-02 15:04"), h.Username, h.Via, len(h.Changes), desc)
+				}
+			})
+		},
+	}
+	list.Flags().IntVar(&limit, "limit", 30, "how many")
+	cmd.AddCommand(list)
+	cmd.AddCommand(&cobra.Command{
+		Use: "show ID", Short: "Show a change: each object before and after", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var h model.HistoryEntry
+			if err := c.Get(ctx(), "/history/"+args[0], nil, &h); err != nil {
+				return err
+			}
+			return output(h, func(w io.Writer) {
+				fmt.Fprintf(w, "%d\t%s by %s (%s): %s\n", h.ID, h.Time.Local().Format("2006-01-02 15:04:05"), h.Username, h.Via, h.Description)
+				for _, ch := range h.Changes {
+					fmt.Fprintf(w, "\n%s %s %s\n", ch.Action(), ch.Kind, ch.Label)
+					for _, line := range diffJSON(ch.Before, ch.After) {
+						fmt.Fprintln(w, "  "+line)
+					}
+				}
+			})
+		},
+	})
+	cmd.AddCommand(&cobra.Command{
+		Use: "diff ID", Short: "Show what a change did to the ruleset", Args: cobra.ExactArgs(1),
+		Long: `Shows the ruleset's diff for a change: the configuration before and after
+it, rendered by this version of Housecarl.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var d struct {
+				Diff string `json:"diff"`
+			}
+			if err := c.Get(ctx(), "/history/"+args[0]+"/diff", nil, &d); err != nil {
+				return err
+			}
+			return output(d, func(w io.Writer) {
+				if d.Diff == "" {
+					fmt.Fprintln(w, "the ruleset didn't change")
+					return
+				}
+				printDiff(w, d.Diff)
+			})
+		},
+	})
+	cmd.AddCommand(&cobra.Command{
+		Use:   "undo ID",
+		Short: "Undo a change",
+		Long: `Undoes a change by applying it backwards. It is refused when an object the
+change touched has changed since; undo the later changes first.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var resp api.MutationResponse
+			if err := c.Do(ctx(), "POST", "/history/"+args[0]+"/revert", nil, nil, &resp); err != nil {
+				return err
+			}
+			return report(&resp)
+		},
+	})
+	return cmd
+}
+
+// diffJSON lists the fields that differ between two versions of an
+// object: "name: office -> lab". An object within (settings' builtins)
+// is compared field by field: "builtins.drop_invalid: false -> true".
+func diffJSON(before, after json.RawMessage) []string {
+	var a, b map[string]any
+	json.Unmarshal(before, &a)
+	json.Unmarshal(after, &b)
+	a, b = flatten(a, "", map[string]any{}), flatten(b, "", map[string]any{})
+	keys := map[string]bool{}
+	for k := range a {
+		keys[k] = true
+	}
+	for k := range b {
+		keys[k] = true
+	}
+	var names []string
+	for k := range keys {
+		names = append(names, k)
+	}
+	sort.Strings(names)
+	var out []string
+	show := func(v any, ok bool) string {
+		if !ok {
+			return "-"
+		}
+		x, _ := json.Marshal(v)
+		return string(x)
+	}
+	for _, k := range names {
+		va, oka := a[k]
+		vb, okb := b[k]
+		sa, sb := show(va, oka), show(vb, okb)
+		if sa == sb {
+			continue
+		}
+		switch {
+		case before == nil:
+			out = append(out, k+": "+sb)
+		case after == nil:
+			out = append(out, k+": "+sa)
+		default:
+			out = append(out, k+": "+sa+" -> "+sb)
+		}
+	}
+	return out
+}
+
+// flatten lifts the fields of objects within m to dotted keys; lists
+// stay whole.
+func flatten(m map[string]any, prefix string, out map[string]any) map[string]any {
+	for k, v := range m {
+		if o, ok := v.(map[string]any); ok && len(o) > 0 {
+			flatten(o, prefix+k+".", out)
+		} else {
+			out[prefix+k] = v
+		}
+	}
+	return out
+}
+
+func whoamiCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "whoami", Short: "Show who you are to the server", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var me api.Whoami
+			if err := c.Get(ctx(), "/auth/whoami", nil, &me); err != nil {
+				return err
+			}
+			return output(me, func(w io.Writer) {
+				fmt.Fprintf(w, "User:\t%s (%s)\n", me.User.Username, me.Method)
+				fmt.Fprintf(w, "Administrator:\t%v\n", me.Admin)
+				fmt.Fprintf(w, "May:\t%s\n", strings.Join(me.Permissions, ", "))
+				fmt.Fprintf(w, "Server:\tHousecarl %s\n", me.Build.String())
+			})
+		},
+	}
+}
+
+func readSecret(prompt string) (string, error) {
+	fmt.Fprint(os.Stderr, prompt)
+	if term.IsTerminal(int(os.Stdin.Fd())) {
+		b, err := term.ReadPassword(int(os.Stdin.Fd()))
+		fmt.Fprintln(os.Stderr)
+		return strings.TrimSpace(string(b)), err
+	}
+	var s string
+	_, err := fmt.Fscanln(os.Stdin, &s)
+	return strings.TrimSpace(s), err
+}
+
+func userID(name string) (int64, error) {
+	c, err := newClient()
+	if err != nil {
+		return 0, err
+	}
+	var us []api.User
+	if err := c.Get(ctx(), "/users", nil, &us); err != nil {
+		return 0, err
+	}
+	for _, u := range us {
+		if strings.EqualFold(u.Username, name) || strconv.FormatInt(u.ID, 10) == name {
+			return u.ID, nil
+		}
+	}
+	return 0, fmt.Errorf("no user %s", name)
+}
+
+func userCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "user", Aliases: []string{"users"}, Short: "Users (administrators only)"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List users", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var us []api.User
+			if err := c.Get(ctx(), "/users", nil, &us); err != nil {
+				return err
+			}
+			return output(us, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tUSER\tADMIN\tGRANTS\tPASSWORD\tDISABLED\tLAST LOGIN")
+				for _, u := range us {
+					login := "-"
+					if u.LastLoginAt != nil {
+						login = fmtWhen(*u.LastLoginAt)
+					}
+					fmt.Fprintf(w, "%d\t%s\t%v\t%s\t%v\t%v\t%s\n", u.ID, u.Username, u.Admin, orDash(strings.Join(u.GrantStrings, " ")), u.HasPassword, u.Disabled, login)
+				}
+			})
+		},
+	})
+	var admin, noPassword bool
+	var role, comment string
+	var grants []string
+	add := &cobra.Command{
+		Use:   "add USER",
+		Short: "Add a user (asks for the password)",
+		Long: `Adds a user. Without --admin, give what they may do: a role (viewer,
+operator, log) or grants such as --grant config:read --grant sets:write:guests.
+--no-password makes a service account that only uses API tokens.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			in := api.UserInput{Username: &args[0], Admin: &admin}
+			if !noPassword {
+				pw, err := readSecret("Password for " + args[0] + ": ")
+				if err != nil {
+					return err
+				}
+				in.Password = &pw
+			}
+			if role != "" {
+				in.Role = &role
+			}
+			if grants != nil {
+				in.Grants = &grants
+			}
+			if comment != "" {
+				in.Comment = &comment
+			}
+			var u api.User
+			if err := c.Do(ctx(), "POST", "/users", nil, in, &u); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("added user %s (%d)\n", u.Username, u.ID)
+			}
+			return nil
+		},
+	}
+	f := add.Flags()
+	f.BoolVar(&admin, "admin", false, "an administrator (may do everything)")
+	f.StringVar(&role, "role", "", "a role: viewer, operator or log")
+	f.StringArrayVar(&grants, "grant", nil, "a grant, resource:access (config, sets, log, audit; read or write; sets:write:SET,SET)")
+	f.BoolVar(&noPassword, "no-password", false, "no password: API tokens only")
+	f.StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+
+	var setPassword, disable, enable bool
+	update := &cobra.Command{
+		Use: "update USER", Short: "Change a user", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			id, err := userID(args[0])
+			if err != nil {
+				return err
+			}
+			c, _ := newClient()
+			in := api.UserInput{}
+			fl := cmd.Flags()
+			if fl.Changed("admin") {
+				in.Admin = &admin
+			}
+			if fl.Changed("role") {
+				in.Role = &role
+			}
+			if fl.Changed("grant") {
+				in.Grants = &grants
+			}
+			if fl.Changed("comment") {
+				in.Comment = &comment
+			}
+			if disable || enable {
+				in.Disabled = &disable
+			}
+			if setPassword {
+				pw, err := readSecret("New password for " + args[0] + ": ")
+				if err != nil {
+					return err
+				}
+				in.Password = &pw
+			}
+			var u api.User
+			if err := c.Do(ctx(), "PATCH", "/users/"+strconv.FormatInt(id, 10), nil, in, &u); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("updated user %s\n", u.Username)
+			}
+			return nil
+		},
+	}
+	f = update.Flags()
+	f.BoolVar(&admin, "admin", false, "make (or with --admin=false, unmake) an administrator")
+	f.StringVar(&role, "role", "", "replace the grants with a role's")
+	f.StringArrayVar(&grants, "grant", nil, "replace the grants (repeat for several)")
+	f.BoolVar(&setPassword, "password", false, "set a new password (asks for it)")
+	f.BoolVar(&disable, "disable", false, "disable the user (logs them out)")
+	f.BoolVar(&enable, "enable", false, "enable the user again")
+	f.StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(update)
+	var yes bool
+	del := &cobra.Command{
+		Use: "delete USER", Short: "Delete a user and their tokens", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !confirm(fmt.Sprintf("Delete user %s and their tokens?", args[0]), yes) {
+				return fmt.Errorf("not deleted")
+			}
+			id, err := userID(args[0])
+			if err != nil {
+				return err
+			}
+			c, _ := newClient()
+			if err := c.Do(ctx(), "DELETE", "/users/"+strconv.FormatInt(id, 10), nil, nil, nil); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Println("deleted user", args[0])
+			}
+			return nil
+		},
+	}
+	del.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	cmd.AddCommand(del)
+	return cmd
+}
+
+func tokenCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "token", Aliases: []string{"tokens"}, Short: "API tokens"}
+	var all bool
+	var user string
+	list := &cobra.Command{
+		Use: "list", Short: "List your tokens (--all: everyone's, for administrators)", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			path := "/me/tokens"
+			if all {
+				path = "/tokens"
+			}
+			var ts []api.Token
+			if err := c.Get(ctx(), path, nil, &ts); err != nil {
+				return err
+			}
+			return output(ts, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tUSER\tNAME\tGRANTS\tEXPIRES\tLAST USED")
+				for _, t := range ts {
+					exp, used := "never", "never"
+					if t.ExpiresAt != nil {
+						exp = fmtWhen(*t.ExpiresAt)
+					}
+					if t.LastUsedAt != nil {
+						used = fmtWhen(*t.LastUsedAt) + " from " + t.LastUsedIP
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\n", t.ID, t.Username, t.Name, strings.Join(t.GrantStrings, " "), exp, used)
+				}
+			})
+		},
+	}
+	list.Flags().BoolVar(&all, "all", false, "everyone's tokens")
+	cmd.AddCommand(list)
+	var role, expires string
+	var grants []string
+	var save bool
+	create := &cobra.Command{
+		Use:   "create NAME",
+		Short: "Make an API token (the secret is shown once)",
+		Long: `Makes an API token. It has all of its user's permissions unless --role or
+--grant narrows it. On the server as root, --user names whose token it is.
+--save puts it in the CLI's configuration.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			path := "/me/tokens"
+			if user != "" {
+				id, err := userID(user)
+				if err != nil {
+					return err
+				}
+				path = "/users/" + strconv.FormatInt(id, 10) + "/tokens"
+			}
+			in := api.TokenInput{Name: args[0], Role: role, Grants: grants, ExpiresIn: expires}
+			var t api.Token
+			if err := c.Do(ctx(), "POST", path, nil, in, &t); err != nil {
+				return err
+			}
+			if save {
+				cfg, err := loadCLIConfig()
+				if err != nil {
+					return err
+				}
+				cfg.Token = t.Secret
+				if err := saveCLIConfig(cfg); err != nil {
+					return err
+				}
+				fmt.Fprintln(os.Stderr, "saved in", flagConfig)
+			}
+			if flagOutput == "json" || flagOutput == "yaml" {
+				return output(t, nil)
+			}
+			fmt.Println(t.Secret)
+			if !flagQuiet {
+				fmt.Fprintln(os.Stderr, "this is the only time the token is shown; keep it safe")
+			}
+			return nil
+		},
+	}
+	f := create.Flags()
+	f.StringVar(&role, "role", "", "narrow it to a role: viewer, operator or log")
+	f.StringArrayVar(&grants, "grant", nil, "narrow it to grants (resource:access)")
+	f.StringVar(&expires, "expires", "", "expire after this long (90d, 720h)")
+	f.StringVar(&user, "user", "", "the user it belongs to (administrators)")
+	f.BoolVar(&save, "save", false, "save it as this CLI's token")
+	cmd.AddCommand(create)
+	var grace, rexp string
+	var current, rsave bool
+	rotate := &cobra.Command{
+		Use:   "rotate [ID]",
+		Short: "Give a token a new secret (shown once)",
+		Long: `Gives a token a new secret. It keeps its ID, name and permissions, so only
+the programs that use it need the new secret. The old secret stops at once,
+or after --grace (up to 7d) so you can update them without an outage. A
+token that expires gets its lifetime again, from now, unless --expires says
+otherwise (90d, or never).
+
+  hc token rotate --current --save            # this CLI's own token`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			id := ""
+			switch {
+			case current && len(args) == 0:
+				var me api.Whoami
+				if err := c.Get(ctx(), "/auth/whoami", nil, &me); err != nil {
+					return err
+				}
+				if me.TokenID == 0 {
+					return fmt.Errorf("this CLI isn't using a token (%s); give the token's ID", me.Method)
+				}
+				id = strconv.FormatInt(me.TokenID, 10)
+			case len(args) == 1 && !current:
+				id = args[0]
+			default:
+				return fmt.Errorf("give the token's ID, or --current")
+			}
+			body := api.TokenRotation{Grace: grace}
+			switch rexp {
+			case "":
+			case "never":
+				body.NoExpiry = true
+			default:
+				body.ExpiresIn = rexp
+			}
+			var t api.Token
+			if err := c.Do(ctx(), "POST", "/tokens/"+id+"/rotate", nil, body, &t); err != nil {
+				return err
+			}
+			if rsave {
+				cfg, err := loadCLIConfig()
+				if err != nil {
+					return err
+				}
+				cfg.Token = t.Secret
+				if err := saveCLIConfig(cfg); err != nil {
+					return err
+				}
+				fmt.Fprintln(os.Stderr, "saved in", flagConfig)
+			}
+			if flagOutput == "json" || flagOutput == "yaml" {
+				return output(t, nil)
+			}
+			fmt.Println(t.Secret)
+			if !flagQuiet {
+				if t.PrevExpiresAt != nil {
+					fmt.Fprintf(os.Stderr, "the old secret works until %s\n", t.PrevExpiresAt.Local().Format("2006-01-02 15:04"))
+				} else {
+					fmt.Fprintln(os.Stderr, "the old secret no longer works")
+				}
+			}
+			return nil
+		},
+	}
+	f = rotate.Flags()
+	f.StringVar(&grace, "grace", "", "keep the old secret working this long (30m, 24h, up to 7d)")
+	f.StringVar(&rexp, "expires", "", "a new lifetime from now (90d), or never")
+	f.BoolVar(&current, "current", false, "rotate the token this CLI uses")
+	f.BoolVar(&rsave, "save", false, "save the new secret as this CLI's token")
+	cmd.AddCommand(rotate)
+	cmd.AddCommand(&cobra.Command{
+		Use: "revoke ID", Short: "Revoke a token", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			if err := c.Do(ctx(), "DELETE", "/tokens/"+args[0], nil, nil, nil); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Println("revoked token", args[0])
+			}
+			return nil
+		},
+	})
+	return cmd
+}
+
+func passwordCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "password", Short: "Change your own password", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			cur, err := readSecret("Current password: ")
+			if err != nil {
+				return err
+			}
+			pw, err := readSecret("New password: ")
+			if err != nil {
+				return err
+			}
+			if err := c.Do(ctx(), "POST", "/me/password", nil, api.PasswordChange{Current: cur, Password: pw}, nil); err != nil {
+				return err
+			}
+			fmt.Println("password changed")
+			return nil
+		},
+	}
+}
+
+func auditCmd() *cobra.Command {
+	var user, action, search string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "audit", Short: "Show the audit log: logins, users, tokens, changes, confirmations, set entries", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var evs []model.AuditEvent
+			if err := c.Get(ctx(), "/audit", q("user", user, "action", action, "q", search, "limit", strconv.Itoa(limit)), &evs); err != nil {
+				return err
+			}
+			return output(evs, func(w io.Writer) {
+				fmt.Fprintln(w, "WHEN\tUSER\tFROM\tACTION\tOBJECT\tRESULT\tDETAIL")
+				for _, e := range evs {
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", e.Time.Local().Format("2006-01-02 15:04:05"), orDash(e.Username), orDash(e.IP), e.Action, orDash(e.Object), e.Result, e.Detail)
+				}
+			})
+		},
+	}
+	f := cmd.Flags()
+	f.StringVar(&user, "user", "", "only this user's")
+	f.StringVar(&action, "action", "", "only actions starting with this (auth., config., set.)")
+	f.StringVar(&search, "search", "", "only events whose object or detail contains this")
+	f.IntVar(&limit, "limit", 100, "how many")
+	return cmd
+}
+
+func configCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "config", Short: "This CLI's settings: the server and token"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "show", Short: "Show the CLI's configuration", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := loadCLIConfig()
+			if err != nil {
+				return err
+			}
+			tok := "-"
+			if cfg.Token != "" {
+				tok = cfg.Token[:min(len(cfg.Token), 24)] + "…"
+			}
+			fmt.Printf("file:     %s\nserver:   %s\ntoken:    %s\ninsecure: %v\nca_cert:  %s\n", flagConfig, orDash(cfg.Server)+" (empty: the local socket)", tok, cfg.Insecure, orDash(cfg.CACert))
+			return nil
+		},
+	})
+	cmd.AddCommand(&cobra.Command{
+		Use:   "set KEY [VALUE]",
+		Short: "Set server, token (asks when no value), insecure or ca_cert",
+		Args:  cobra.RangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := loadCLIConfig()
+			if err != nil {
+				return err
+			}
+			val := ""
+			if len(args) == 2 {
+				val = args[1]
+			} else if args[0] == "token" {
+				if val, err = readSecret("API token: "); err != nil {
+					return err
+				}
+			}
+			switch args[0] {
+			case "server":
+				cfg.Server = val
+			case "token":
+				cfg.Token = val
+			case "insecure":
+				cfg.Insecure = val == "true" || val == "yes" || val == "on"
+			case "ca_cert", "ca-cert":
+				cfg.CACert = val
+			default:
+				return fmt.Errorf("unknown setting %q (use server, token, insecure or ca_cert)", args[0])
+			}
+			if err := saveCLIConfig(cfg); err != nil {
+				return err
+			}
+			fmt.Fprintln(os.Stderr, "saved in", flagConfig)
+			return nil
+		},
+	})
+	return cmd
+}

cmd/hc/bans.go

@@ -0,0 +1,313 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+// ---- ban lists (docs/bans.md)
+
+// banTime writes a ban list's time setting: 0 is the default, -1 none.
+func banTime(n int, def string) string {
+	switch {
+	case n < 0:
+		return "no limit"
+	case n == 0:
+		return def + " (default)"
+	}
+	return fmtSeconds(n)
+}
+
+// parseBanTime reads a time flag: a duration, or "none" (-1) for no
+// limit.
+func parseBanTime(s string) (int, error) {
+	if strings.EqualFold(s, "none") || s == "-1" {
+		return -1, nil
+	}
+	return parseSeconds(s)
+}
+
+func banlistCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "banlist", Aliases: []string{"banlists"}, Short: "Ban lists: where Guard and fail2ban ban addresses"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the ban lists", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			bs, err := get[[]model.BanList]("/banlists", nil)
+			if err != nil {
+				return err
+			}
+			bans, _ := get[[]model.Ban]("/bans", nil)
+			count := map[int64]int{}
+			for _, b := range bans {
+				count[b.ListID]++
+			}
+			return output(bs, func(w io.Writer) {
+				fmt.Fprintln(w, "NAME\tSCOPE\tHOSTS\tBANS\tCOMMENT")
+				for _, b := range bs {
+					scope := b.Scope
+					if b.Disabled {
+						scope += " (disabled)"
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n", b.Name, scope, list(b.Hosts), count[b.ID], b.Comment)
+				}
+			})
+		},
+	})
+	cmd.AddCommand(showCmd("banlists", "banlist", func(w io.Writer, b model.BanList) {
+		fmt.Fprintf(w, "Name:\t%s\nScope:\t%s\n", b.Name, b.Scope)
+		if len(b.Hosts) > 0 {
+			fmt.Fprintf(w, "Hosts:\t%s\n", list(b.Hosts))
+		}
+		fmt.Fprintf(w, "Never:\t%s\n", list(b.Never))
+		fmt.Fprintf(w, "Ban time:\t%s\nLongest:\t%s\n", banTime(b.DefaultTime, "1h"), banTime(b.MaxTime, "7d"))
+		fmt.Fprintf(w, "Most bans:\t%d\nIPv6 banned by:\t/%d\n", b.Entries(), b.V6Group())
+		if b.Disabled {
+			fmt.Fprintln(w, "Disabled:\tyes")
+		}
+		if b.Comment != "" {
+			fmt.Fprintf(w, "Comment:\t%s\n", b.Comment)
+		}
+	}))
+	var scope, comment, banT, maxT string
+	var hosts, never []string
+	var entries, group int
+	var disabled bool
+	setFields := func(fl interface{ Changed(string) bool }, p patch) error {
+		if fl.Changed("scope") {
+			p["scope"] = scope
+		}
+		if fl.Changed("host") {
+			p["hosts"] = hosts
+		}
+		if fl.Changed("never") {
+			p["never"] = never
+		}
+		for _, t := range []struct{ flag, field, v string }{{"time", "default_time", banT}, {"max-time", "max_time", maxT}} {
+			if fl.Changed(t.flag) {
+				n, err := parseBanTime(t.v)
+				if err != nil {
+					return err
+				}
+				p[t.field] = n
+			}
+		}
+		if fl.Changed("max-entries") {
+			p["max_entries"] = entries
+		}
+		if fl.Changed("group-v6") {
+			p["group_v6"] = group
+		}
+		if fl.Changed("comment") {
+			p["comment"] = comment
+		}
+		if fl.Changed("disabled") {
+			p["disabled"] = disabled
+		}
+		return nil
+	}
+	flags := func(c *cobra.Command) {
+		f := c.Flags()
+		f.StringVar(&scope, "scope", "", "what bans keep addresses from: firewall, hosts or everything")
+		f.StringArrayVar(&hosts, "host", nil, "scope hosts: a machine's address, or a set (repeat)")
+		f.StringArrayVar(&never, "never", nil, "an address, prefix or set never banned (repeat)")
+		f.StringVar(&banT, "time", "", "a ban's length when none is asked for (default 1h)")
+		f.StringVar(&maxT, "max-time", "", "the longest ban (default 7d; none for no limit)")
+		f.IntVar(&entries, "max-entries", 0, "the most bans the list holds (default 10000)")
+		f.IntVar(&group, "group-v6", 0, "IPv6 prefix length banned for an address (default 64)")
+		f.StringVar(&comment, "comment", "", "a comment")
+		f.BoolVar(&disabled, "disabled", false, "the list's bans aren't in force")
+	}
+	add := &cobra.Command{
+		Use: "add NAME", Short: "Add a ban list", Args: cobra.ExactArgs(1),
+		Long: `Adds a ban list. A program bans an address into it (Guard, fail2ban's
+housecarl action, hc ban), and the list says what that address is kept
+from:
+
+  hc banlist add router --scope firewall                  # the firewall itself
+  hc banlist add web --scope hosts --host 10.0.10.57 --host 2001:db8:10::57
+  hc banlist add edge --scope everything --never trusted
+
+Give the program a token for its list alone:
+
+  hc user add guard-web --no-password --grant bans:write:web
+  hc token create web1 --user guard-web     # as root on the firewall`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{"name": args[0]}
+			if err := setFields(cmd.Flags(), p); err != nil {
+				return err
+			}
+			return change("POST", "/banlists", p)
+		},
+	}
+	flags(add)
+	cmd.AddCommand(add)
+	update := &cobra.Command{
+		Use: "update BANLIST", Short: "Change a ban list", Args: cobra.ExactArgs(1),
+		Long: `Changes a ban list; --host and --never replace the lists. A ban list
+can't be renamed: tokens and programs know it by its name.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{}
+			if err := setFields(cmd.Flags(), p); err != nil {
+				return err
+			}
+			return change("PATCH", objPath("banlists", args[0]), p)
+		},
+	}
+	flags(update)
+	cmd.AddCommand(update)
+	cmd.AddCommand(deleteCmd("banlists", "banlist"))
+	return cmd
+}
+
+// bansCmd lists the active bans.
+func bansCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "bans [BANLIST]", Short: "List the active bans", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			path := "/bans"
+			if len(args) == 1 {
+				path = objPath("banlists", args[0]) + "/bans"
+			}
+			bans, err := get[[]model.Ban](path, nil)
+			if err != nil {
+				return err
+			}
+			return output(bans, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tLIST\tADDRESS\tPORT\tENDS IN\tBY\tSOURCE\tREASON")
+				for _, b := range bans {
+					port := "all"
+					if b.Protocol != "" {
+						port = fmt.Sprintf("%s/%d", b.Protocol, b.Port)
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", b.ID, b.List, b.Address, port, fmtSeconds(b.Timeout), b.Who, orDash(b.Source), orDash(clipText(b.Reason, 60)))
+				}
+			})
+		},
+	}
+}
+
+func clipText(s string, n int) string {
+	if len(s) > n {
+		return s[:n-1] + "…"
+	}
+	return s
+}
+
+// banCmd bans an address by hand.
+func banCmd() *cobra.Command {
+	var tcp, udp []string
+	var t, reason string
+	cmd := &cobra.Command{
+		Use: "ban BANLIST ADDRESS", Short: "Ban an address", Args: cobra.ExactArgs(2),
+		Long: `Bans an address in a ban list: every port, or with --tcp and --udp some
+(numbers or service names). IPv6 addresses are banned with the list's
+grouping (a /64 unless set otherwise).
+
+  hc ban edge 203.0.113.9 --time 1d
+  hc ban web 203.0.113.9 --tcp ssh --tcp 8443 --reason "tried passwords"`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := api.BanInput{Address: args[1], Reason: reason, Source: "hc"}
+			if t != "" {
+				n, err := parseBanTime(t)
+				if err != nil {
+					return err
+				}
+				in.Time = n
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var out api.BanResult
+			do := func(proto string, ports []string) error {
+				in.Protocol = proto
+				in.Ports = nil
+				if len(ports) > 0 {
+					in.Ports = []byte(strconv.Quote(strings.Join(ports, ",")))
+				}
+				var res api.BanResult
+				if err := c.Do(ctx(), "POST", objPath("banlists", args[0])+"/bans", nil, in, &res); err != nil {
+					return err
+				}
+				out.Bans = append(out.Bans, res.Bans...)
+				out.Closed += res.Closed
+				return nil
+			}
+			switch {
+			case len(tcp) == 0 && len(udp) == 0:
+				err = do("", nil)
+			default:
+				if len(tcp) > 0 {
+					err = do("tcp", tcp)
+				}
+				if err == nil && len(udp) > 0 {
+					err = do("udp", udp)
+				}
+			}
+			if err != nil {
+				return err
+			}
+			return output(out, func(w io.Writer) {
+				for _, b := range out.Bans {
+					what := "banned"
+					if b.Existing {
+						what = "renewed"
+					}
+					port := "every port"
+					if b.Protocol != "" {
+						port = fmt.Sprintf("%s/%d", b.Protocol, b.Port)
+					}
+					ends := "never ends"
+					if b.Timeout > 0 {
+						ends = "ends in " + fmtSeconds(b.Timeout)
+					}
+					fmt.Fprintf(w, "%s %s, %s, in %s (%s)\n", what, b.Address, port, b.List, ends)
+				}
+				if out.Closed > 0 {
+					fmt.Fprintf(w, "%d connections closed\n", out.Closed)
+				}
+			})
+		},
+	}
+	f := cmd.Flags()
+	f.StringArrayVar(&tcp, "tcp", nil, "ban only this TCP port or service (repeat)")
+	f.StringArrayVar(&udp, "udp", nil, "ban only this UDP port or service (repeat)")
+	f.StringVar(&t, "time", "", "how long (1h, 7d; default the list's; none for ever, if the list allows)")
+	f.StringVar(&reason, "reason", "", "why, for the record")
+	return cmd
+}
+
+// unbanCmd lifts bans: by ID, or every ban of an address.
+func unbanCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "unban BANLIST ADDRESS|ID", Short: "Lift a ban, or all of an address's", Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			path, query := objPath("banlists", args[0])+"/bans", url.Values{}
+			if _, err := strconv.ParseInt(args[1], 10, 64); err == nil {
+				path += "/" + args[1]
+			} else {
+				query.Set("address", args[1])
+			}
+			var out map[string]int
+			if err := c.Do(ctx(), "DELETE", path, query, nil, &out); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("%d bans lifted\n", out["removed"])
+			}
+			return nil
+		},
+	}
+}

cmd/hc/cert.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+)
+
+func certCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "cert",
+		Aliases: []string{"certificate"},
+		Short:   "The web UI's certificate, obtained through Jarl (administrators)",
+		Long: `Shows the certificate housecarld obtains itself for its web UI (tls.acme in
+housecarl.yaml, with Jarl publishing the DNS-01 challenge; docs/certificates.md).
+'check' tests the whole path without asking the CA; 'renew' gets a new
+certificate now.`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error { return certStatus() },
+	}
+	cmd.AddCommand(&cobra.Command{Use: "status", Short: "Show the certificate", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error { return certStatus() }})
+	cmd.AddCommand(&cobra.Command{
+		Use:   "check",
+		Short: "Check that the challenge can be published and seen, without the CA",
+		Long: `Runs the check housecarld runs before every certificate order: Jarl answers
+and knows the token, a zone holds _acme-challenge.DOMAIN, the token may
+write it, a test record is written, the zone's name servers and the public
+resolvers serve it, and it is removed again. It can take a minute or two.`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			c.HTTP.Timeout = 5 * time.Minute
+			if !flagQuiet && flagOutput == "table" {
+				fmt.Fprintln(os.Stderr, "checking (this can take a minute or two)…")
+			}
+			var ch api.CertCheck
+			if err := c.Do(ctx(), "POST", "/certificate/check", nil, nil, &ch); err != nil {
+				return err
+			}
+			if err := output(ch, func(w io.Writer) { printCheck(w, &ch) }); err != nil {
+				return err
+			}
+			if !ch.OK {
+				return fmt.Errorf("the check failed")
+			}
+			return nil
+		},
+	})
+	var wait bool
+	renew := &cobra.Command{
+		Use:   "renew",
+		Short: "Get a new certificate now (the check runs first)",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var r api.CertificateResponse
+			if err := c.Do(ctx(), "POST", "/certificate/renew", nil, nil, &r); err != nil {
+				return err
+			}
+			if !wait {
+				fmt.Println("renewing in the background; follow it with 'hc cert'")
+				return nil
+			}
+			start := time.Now()
+			for {
+				time.Sleep(3 * time.Second)
+				if err := c.Get(ctx(), "/certificate", nil, &r); err != nil {
+					return err
+				}
+				st := r.Status
+				if st.State != "working" && st.LastAttempt != nil && !st.LastAttempt.Before(start.Add(-time.Second)) {
+					break
+				}
+			}
+			if err := certStatus(); err != nil {
+				return err
+			}
+			if r.Status.LastError != "" {
+				return fmt.Errorf("renewing failed")
+			}
+			return nil
+		},
+	}
+	renew.Flags().BoolVarP(&wait, "wait", "w", false, "wait until it is done")
+	cmd.AddCommand(renew)
+	return cmd
+}
+
+func certStatus() error {
+	c, err := newClient()
+	if err != nil {
+		return err
+	}
+	var r api.CertificateResponse
+	if err := c.Get(ctx(), "/certificate", nil, &r); err != nil {
+		return err
+	}
+	return output(r, func(w io.Writer) {
+		switch r.Source {
+		case "file":
+			fmt.Fprintln(w, "Source:\ttls.cert_file (renewed by you)")
+			return
+		case "none":
+			fmt.Fprintln(w, "Source:\tnone: the web UI is plain HTTP (set tls.acme or tls.cert_file in housecarl.yaml; docs/certificates.md)")
+			return
+		}
+		st := r.Status
+		fmt.Fprintf(w, "Domain:\t%s\n", st.Domain)
+		ca := st.Directory
+		if st.Staging {
+			ca += " (staging: browsers don't trust it)"
+		}
+		fmt.Fprintf(w, "CA:\t%s\n", ca)
+		fmt.Fprintf(w, "State:\t%s\n", st.State)
+		if st.NotAfter != nil {
+			fmt.Fprintf(w, "Valid:\t%s to %s, from %s\n", st.NotBefore.Local().Format("2006-01-02"), st.NotAfter.Local().Format("2006-01-02 15:04"), st.Issuer)
+			fmt.Fprintf(w, "Renews:\t%s\n", st.RenewAt.Local().Format("2006-01-02"))
+		}
+		if st.LastAttempt != nil {
+			fmt.Fprintf(w, "Last attempt:\t%s\n", st.LastAttempt.Local().Format("2006-01-02 15:04"))
+		}
+		if st.NextAttempt != nil && st.State != "working" {
+			fmt.Fprintf(w, "Next check:\t%s\n", st.NextAttempt.Local().Format("2006-01-02 15:04"))
+		}
+		if st.LastError != "" {
+			fmt.Fprintf(w, "Error:\t%s\n", st.LastError)
+		}
+		if st.Check != nil {
+			fmt.Fprintf(w, "\nLast check, %s:\n", st.Check.Time.Local().Format("2006-01-02 15:04"))
+			printCheck(w, st.Check)
+		}
+	})
+}
+
+func printCheck(w io.Writer, c *api.CertCheck) {
+	for _, s := range c.Steps {
+		mark := map[string]string{"ok": "ok  ", "fail": "FAIL", "warn": "warn", "skip": "skip"}[s.Status]
+		fmt.Fprintf(w, "  %s\t%s\t%s\n", mark, s.Name, s.Detail)
+		if s.Status == "fail" && len(s.Queries) > 0 {
+			for _, q := range s.Queries {
+				fmt.Fprintf(w, "    \t\t%s\n", strings.TrimSpace(q))
+			}
+		}
+	}
+}

cmd/hc/changesets.go

@@ -0,0 +1,377 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+	"housecarl/internal/client"
+	"housecarl/internal/model"
+)
+
+// Changesets: configuration changes staged to be reviewed and applied
+// together. -C/--changeset stages into a changeset as given (housecarld
+// still refuses one that isn't open, or another user's unless you are an
+// administrator). One from HOUSECARL_CHANGESET is checked first: an
+// exported ID outlives its changeset (it gets applied), and IDs keep
+// growing, so a stale or mistyped one could name someone else's later. It
+// is used only when it is open and yours, and hc says on stderr which
+// changeset that is.
+
+var flagChangeset int64
+
+// envChangeset returns the changeset in HOUSECARL_CHANGESET (0 if unset).
+func envChangeset() (int64, error) {
+	v := os.Getenv("HOUSECARL_CHANGESET")
+	if v == "" {
+		return 0, nil
+	}
+	n, err := strconv.ParseInt(v, 10, 64)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("invalid HOUSECARL_CHANGESET %q", v)
+	}
+	return n, nil
+}
+
+const startHint = "export HOUSECARL_CHANGESET=$(hc changeset create -q)"
+
+// checkEnvChangeset checks the changeset from HOUSECARL_CHANGESET before
+// it is used. With mustOwn, it must also be open and the caller's own
+// (for staging into it, applying or discarding it).
+func checkEnvChangeset(c *client.Client, id int64, mustOwn bool) (*model.Changeset, error) {
+	var cs model.Changeset
+	if err := c.Do(ctx(), "GET", fmt.Sprintf("/changesets/%d", id), nil, nil, &cs); err != nil {
+		var ce *client.Error
+		if errors.As(err, &ce) && ce.Status == 404 {
+			return nil, fmt.Errorf("HOUSECARL_CHANGESET is %d, but there is no changeset %d: start one with '%s', or 'unset HOUSECARL_CHANGESET' to make changes at once", id, id, startHint)
+		}
+		return nil, err
+	}
+	if mustOwn {
+		if cs.Status != model.ChangesetOpen {
+			return nil, fmt.Errorf("HOUSECARL_CHANGESET is %d, but changeset %d is %s: 'unset HOUSECARL_CHANGESET' to make changes at once, or start a new one with '%s'", id, id, cs.Status, startHint)
+		}
+		var me api.Whoami
+		if err := c.Do(ctx(), "GET", "/auth/whoami", nil, nil, &me); err != nil {
+			return nil, err
+		}
+		mine := me.User != nil && cs.UserID == me.User.ID && (me.User.ID != 0 || cs.Username == me.User.Username)
+		if !mine {
+			return nil, fmt.Errorf("HOUSECARL_CHANGESET is %d, but changeset %d is %s's (made via %s), not yours, so it isn't used from HOUSECARL_CHANGESET; give it as -C %d to use it anyway, or 'unset HOUSECARL_CHANGESET'", id, id, cs.Username, cs.Via, id)
+		}
+	}
+	if !flagQuiet {
+		desc := cs.Description
+		if desc == "" {
+			desc = "no description"
+		}
+		fmt.Fprintf(os.Stderr, "changeset %d (%s, %s) from HOUSECARL_CHANGESET\n", id, desc, plural(len(cs.Ops), "operation"))
+	}
+	return &cs, nil
+}
+
+// setupChangeset makes the client stage configuration changes: in
+// --changeset's changeset, else HOUSECARL_CHANGESET's (checked before
+// the first change).
+func setupChangeset(c *client.Client) error {
+	if flagChangeset > 0 {
+		c.Changeset = flagChangeset
+		return nil
+	}
+	n, err := envChangeset()
+	if err != nil || n == 0 {
+		return err
+	}
+	c.Changeset = n
+	c.BeforeStage = func() error {
+		_, err := checkEnvChangeset(c, n, true)
+		return err
+	}
+	return nil
+}
+
+// changesetArg returns the changeset a changeset subcommand works on: the
+// ID argument, else --changeset, else HOUSECARL_CHANGESET (checked).
+// fromEnv says it came from HOUSECARL_CHANGESET.
+func changesetArg(c *client.Client, args []string, mustOwn bool) (id string, fromEnv bool, err error) {
+	if len(args) > 0 {
+		return args[0], false, nil
+	}
+	if flagChangeset > 0 {
+		return strconv.FormatInt(flagChangeset, 10), false, nil
+	}
+	n, err := envChangeset()
+	if err != nil {
+		return "", false, err
+	}
+	if n == 0 {
+		return "", false, errors.New("which changeset? Give its ID, or set HOUSECARL_CHANGESET (" + startHint + ")")
+	}
+	if _, err := checkEnvChangeset(c, n, mustOwn); err != nil {
+		return "", false, err
+	}
+	return strconv.FormatInt(n, 10), true, nil
+}
+
+// envChangesetDone reminds that HOUSECARL_CHANGESET still names a
+// changeset that was just applied or discarded.
+func envChangesetDone(id, what string) {
+	if !flagQuiet {
+		fmt.Fprintf(os.Stderr, "HOUSECARL_CHANGESET still names changeset %s, which is now %s: run 'unset HOUSECARL_CHANGESET' (or export a new one) before making more changes\n", id, what)
+	}
+}
+
+func plural(n int, word string) string {
+	if n == 1 {
+		return "1 " + word
+	}
+	return strconv.Itoa(n) + " " + word + "s"
+}
+
+// printDiff prints a unified diff, colored on a terminal.
+func printDiff(w io.Writer, diff string) {
+	color := isTerminal(os.Stdout)
+	for _, line := range strings.Split(strings.TrimSuffix(diff, "\n"), "\n") {
+		switch {
+		case !color:
+		case strings.HasPrefix(line, "+++"), strings.HasPrefix(line, "---"):
+			line = "\x1b[1m" + line + "\x1b[0m"
+		case strings.HasPrefix(line, "+"):
+			line = "\x1b[32m" + line + "\x1b[0m"
+		case strings.HasPrefix(line, "-"):
+			line = "\x1b[31m" + line + "\x1b[0m"
+		case strings.HasPrefix(line, "@@"):
+			line = "\x1b[36m" + line + "\x1b[0m"
+		}
+		fmt.Fprintln(w, line)
+	}
+}
+
+func isTerminal(f *os.File) bool {
+	fi, err := f.Stat()
+	return err == nil && fi.Mode()&os.ModeCharDevice != 0
+}
+
+func changesetCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "changeset",
+		Aliases: []string{"changesets", "cs"},
+		Short:   "Stage changes to review and apply together",
+		Long: `A changeset collects configuration changes to apply together, as one
+change in the history. Each change is checked as it is staged; 'hc
+changeset diff' shows what applying them would do: the problems, nftables'
+check and the ruleset's diff. Over the network an applied changeset is on
+probation like any change.
+
+The web UI stages its edits in a changeset by default; hc and the API
+make changes at once unless one is named with -C ID or HOUSECARL_CHANGESET.
+Changesets staged in the web UI show up here too, and the other way round.`,
+		Example: `  export HOUSECARL_CHANGESET=$(hc changeset create "guest network" -q)
+  hc zone add guest -i eth1.30
+  hc rule add forward --in guest --out wan
+  hc changeset diff                 # HOUSECARL_CHANGESET's
+  hc changeset apply
+  unset HOUSECARL_CHANGESET`,
+	}
+	cmd.AddCommand(&cobra.Command{
+		Use: "create [DESCRIPTION]", Short: "Start a changeset (-q prints only its ID)", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			in := api.ChangesetInput{}
+			if len(args) > 0 {
+				in.Description = args[0]
+			}
+			var cs model.Changeset
+			if err := c.Do(ctx(), "POST", "/changesets", nil, in, &cs); err != nil {
+				return err
+			}
+			if flagQuiet || flagOutput == "id" {
+				fmt.Println(cs.ID)
+				return nil
+			}
+			fmt.Printf("created changeset %d\nstage changes with -C %d (or export HOUSECARL_CHANGESET=%d), then 'hc changeset apply'\n", cs.ID, cs.ID, cs.ID)
+			return nil
+		},
+	})
+	var status string
+	var mine bool
+	list := &cobra.Command{
+		Use: "list", Short: "List changesets (open ones by default)", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var css []*model.Changeset
+			query := q("status", status)
+			if mine {
+				query.Set("mine", "1")
+			}
+			if err := c.Do(ctx(), "GET", "/changesets", query, nil, &css); err != nil {
+				return err
+			}
+			return output(css, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tSTATUS\tBY\tVIA\tOPERATIONS\tUPDATED\tDESCRIPTION")
+				for _, cs := range css {
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\t%s\t%s\n", cs.ID, cs.Status, cs.Username, cs.Via, len(cs.Ops), cs.UpdatedAt.Local().Format("2006-01-02 15:04"), cs.Description)
+				}
+			})
+		},
+	}
+	list.Flags().StringVar(&status, "status", "open", "open, applied, discarded, or \"\" for all")
+	list.Flags().BoolVar(&mine, "mine", false, "only your own")
+	cmd.AddCommand(list)
+	cmd.AddCommand(&cobra.Command{
+		Use: "show [ID]", Short: "Show a changeset's operations (default: -C or HOUSECARL_CHANGESET)", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			id, _, err := changesetArg(c, args, false)
+			if err != nil {
+				return err
+			}
+			var cs model.Changeset
+			if err := c.Do(ctx(), "GET", "/changesets/"+id, nil, nil, &cs); err != nil {
+				return err
+			}
+			return output(&cs, func(w io.Writer) {
+				fmt.Fprintf(w, "Changeset:\t%d (%s)\n", cs.ID, cs.Status)
+				fmt.Fprintf(w, "By:\t%s via %s, %s\n", cs.Username, cs.Via, cs.CreatedAt.Local().Format("2006-01-02 15:04"))
+				if cs.Description != "" {
+					fmt.Fprintf(w, "Description:\t%s\n", cs.Description)
+				}
+				if cs.HistoryID != 0 {
+					fmt.Fprintf(w, "History:\t%d\n", cs.HistoryID)
+				}
+				fmt.Fprintln(w, "\nOP\tCHANGE")
+				for _, o := range cs.Ops {
+					fmt.Fprintf(w, "%d\t%s\n", o.Seq, o.Summary)
+				}
+			})
+		},
+	})
+	cmd.AddCommand(&cobra.Command{
+		Use: "diff [ID]", Short: "Check an open changeset and show what applying it would change (default: -C or HOUSECARL_CHANGESET)", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			id, _, err := changesetArg(c, args, false)
+			if err != nil {
+				return err
+			}
+			var pv api.ChangesetPreview
+			if err := c.Do(ctx(), "GET", "/changesets/"+id+"/preview", nil, nil, &pv); err != nil {
+				return err
+			}
+			if flagOutput == "json" || flagOutput == "yaml" {
+				return output(&pv, nil)
+			}
+			for _, ch := range pv.Changes {
+				if ch.Kind == model.KindSettings {
+					fmt.Printf("would %s settings\n", ch.Action())
+				} else {
+					fmt.Printf("would %s %s %s\n", ch.Action(), ch.Kind, ch.Label)
+				}
+			}
+			printProblems(os.Stderr, pv.Warnings)
+			if pv.Error != nil {
+				printProblems(os.Stderr, pv.Error.Problems)
+				return fmt.Errorf("changeset %s can't be applied: %s", id, pv.Error.Message)
+			}
+			if !pv.Valid {
+				fmt.Println("nothing to change")
+				return nil
+			}
+			fmt.Println()
+			printDiff(os.Stdout, pv.Diff)
+			fmt.Printf("\nvalid: nftables accepts the new ruleset; apply with 'hc changeset apply %s'\n", id)
+			return nil
+		},
+	})
+	var desc string
+	apply := &cobra.Command{
+		Use: "apply [ID]", Short: "Apply an open changeset (default: -C or HOUSECARL_CHANGESET)", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			id, fromEnv, err := changesetArg(c, args, true)
+			if err != nil {
+				return err
+			}
+			var resp api.MutationResponse
+			if err := c.Do(ctx(), "POST", "/changesets/"+id+"/apply", nil, api.ChangesetInput{Description: desc}, &resp); err != nil {
+				return err
+			}
+			if err := report(&resp); err != nil {
+				return err
+			}
+			if fromEnv {
+				envChangesetDone(id, "applied")
+			}
+			return nil
+		},
+	}
+	apply.Flags().StringVar(&desc, "description", "", "the history entry's description (default: the changeset's)")
+	cmd.AddCommand(apply)
+	cmd.AddCommand(&cobra.Command{
+		Use: "discard [ID]", Short: "Throw away an open changeset (default: -C or HOUSECARL_CHANGESET)", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			id, fromEnv, err := changesetArg(c, args, true)
+			if err != nil {
+				return err
+			}
+			if err := c.Do(ctx(), "POST", "/changesets/"+id+"/discard", nil, nil, nil); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("discarded changeset %s\n", id)
+			}
+			if fromEnv {
+				envChangesetDone(id, "discarded")
+			}
+			return nil
+		},
+	})
+	cmd.AddCommand(&cobra.Command{
+		Use: "remove [ID] OP", Short: "Remove one operation from an open changeset (default: -C or HOUSECARL_CHANGESET)", Args: cobra.RangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			op := args[len(args)-1]
+			id, _, err := changesetArg(c, args[:len(args)-1], true)
+			if err != nil {
+				return err
+			}
+			var cs model.Changeset
+			if err := c.Do(ctx(), "DELETE", "/changesets/"+id+"/ops/"+op, nil, nil, &cs); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("removed operation %s; changeset %s has %s\n", op, id, plural(len(cs.Ops), "operation"))
+			}
+			return nil
+		},
+	})
+	return cmd
+}

cmd/hc/firewall.go

@@ -0,0 +1,605 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+func jsonUnmarshal(b []byte, v any) error { return json.Unmarshal(b, v) }
+
+// ---- status
+
+func statusCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "status", Short: "Show the firewall's state: ruleset, probation, problems, interfaces", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			st, err := get[api.Status]("/status", nil)
+			if err != nil {
+				return err
+			}
+			return output(st, func(w io.Writer) {
+				fmt.Fprintf(w, "Housecarl:\t%s, up since %s\n", st.Build.Label, st.Started.Local().Format("2006-01-02 15:04"))
+				if st.Mode == api.ModeShadow {
+					fmt.Fprintf(w, "Mode:\tSHADOW: the ruleset blocks, rejects and translates nothing; the other firewall decides (Housecarl's chains run %s it)\n", st.ShadowPosition)
+				}
+				if st.Applied {
+					fmt.Fprintln(w, "Ruleset:\tin force (table inet housecarl)")
+				} else if st.Unconfigured {
+					fmt.Fprintln(w, "Ruleset:\tnone yet: the configuration is empty (add rules, or 'hc import')")
+				} else {
+					fmt.Fprintf(w, "Ruleset:\tNOT in force: %s\n", st.ApplyError)
+				}
+				if len(st.Probation.Pending) > 0 {
+					n := len(st.Probation.Pending)
+					what := map[bool]string{true: "1 change rolls", false: fmt.Sprintf("%d changes roll", n)}[n == 1]
+					fmt.Fprintf(w, "Probation:\t%s back in %ds unless confirmed ('hc confirm')\n", what, st.Probation.SecondsLeft)
+				}
+				if len(st.OtherTables) > 0 {
+					fmt.Fprintf(w, "Other tables:\t%s (not Housecarl's; 'hc takeover' deletes them once replaced)\n", strings.Join(st.OtherTables, ", "))
+				}
+				var counts []string
+				for _, k := range model.Kinds {
+					counts = append(counts, fmt.Sprintf("%d %ss", st.Counts[k], k))
+				}
+				fmt.Fprintf(w, "Configuration:\t%s\n", strings.Join(counts, ", "))
+				fmt.Fprintln(w, "Interfaces:")
+				for _, i := range st.Interfaces {
+					state := "down"
+					if i.Up {
+						state = "up"
+					}
+					fmt.Fprintf(w, "  %s\t%s\tzone %s\t%s\n", i.Name, state, orDash(i.Zone), strings.Join(i.Addresses, ", "))
+				}
+				if len(st.Problems) > 0 {
+					fmt.Fprintln(w, "Problems:")
+					for _, p := range st.Problems {
+						fmt.Fprintf(w, "  %s:\t%s: %s [%s]\n", p.Severity, p.Object, p.Message, p.Code)
+					}
+				}
+			})
+		},
+	}
+}
+
+func interfaceCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "interfaces", Aliases: []string{"interface"}, Short: "List the machine's interfaces and their zones", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ifs, err := get[[]api.HostInterface]("/interfaces", nil)
+			if err != nil {
+				return err
+			}
+			return output(ifs, func(w io.Writer) {
+				fmt.Fprintln(w, "INTERFACE\tSTATE\tZONE\tADDRESSES")
+				for _, i := range ifs {
+					state := "down"
+					if i.Up {
+						state = "up"
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", i.Name, state, orDash(i.Zone), orDash(strings.Join(i.Addresses, ", ")))
+				}
+			})
+		},
+	}
+}
+
+// ---- settings
+
+func settingsCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "settings", Short: "Policies, built-in rules and other firewall-wide settings"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "show", Short: "Show the settings", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s, err := get[model.Settings]("/settings", nil)
+			if err != nil {
+				return err
+			}
+			return output(s, func(w io.Writer) {
+				fmt.Fprintf(w, "Policies:\tinput %s, forward %s, output %s\n", s.InputPolicy, s.ForwardPolicy, s.OutputPolicy)
+				fmt.Fprintf(w, "Log policy drops:\t%s\n", yesNo(s.LogPolicyDrops))
+				b, _ := json.Marshal(s.Builtins)
+				var m map[string]bool
+				json.Unmarshal(b, &m)
+				var names []string
+				for n := range m {
+					names = append(names, n)
+				}
+				sort.Strings(names)
+				for i, n := range names {
+					label := ""
+					if i == 0 {
+						label = "Built-ins:"
+					}
+					fmt.Fprintf(w, "%s\t%s %s\n", label, n, map[bool]string{true: "on", false: "off"}[m[n]])
+				}
+				fmt.Fprintf(w, "Own prefixes:\t%s\n", list(s.OwnPrefixes))
+				fmt.Fprintf(w, "Antispoof IPv4:\t%s\n", list(s.AntispoofV4))
+				fmt.Fprintf(w, "Antispoof IPv6:\t%s\n", list(s.AntispoofV6))
+				fmt.Fprintf(w, "Flowtable:\t%s\n", list(s.FlowtableDevices))
+				fmt.Fprintf(w, "Probation:\t%s\n", fmtSeconds(s.Probation()))
+				fmt.Fprintf(w, "Traffic per device:\t%s\n", map[bool]string{true: "counted (statistics)", false: "not counted"}[s.StatsPerDevice])
+				if s.ManualConfirm {
+					fmt.Fprintln(w, "Confirmation:\tby a person ('hc confirm', the web UI's banner)")
+				} else {
+					fmt.Fprintln(w, "Confirmation:\tautomatic, by the client that made the change, once it reaches Housecarl again over a new connection")
+				}
+			})
+		},
+	})
+	var input, forward, output, logDrops, probation, autoConf, perDevice string
+	var builtins, own, flowtable, spoof4, spoof6 []string
+	set := &cobra.Command{
+		Use: "set [flags]", Short: "Change settings (only the flags given)", Args: cobra.NoArgs,
+		Long: `Changes the settings. Built-ins are rules Housecarl adds itself; each can be
+turned off: loopback, established, drop_invalid, icmpv6, ping_internal,
+dhcpv6_client, antispoof, mss_clamp.
+
+  hc settings set --builtin mss_clamp=on --own-prefix 2001:db8::/48
+  hc settings set --forward-policy drop --log-drops on
+  hc settings set --flowtable wan --flowtable eth1
+  hc settings set --stats-per-device on`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fl := cmd.Flags()
+			p := patch{}
+			for flag, field := range map[string]string{"input-policy": "input_policy", "forward-policy": "forward_policy", "output-policy": "output_policy"} {
+				if fl.Changed(flag) {
+					v, _ := fl.GetString(flag)
+					p[field] = v
+				}
+			}
+			if fl.Changed("log-drops") {
+				on, err := onOff("log-drops", logDrops)
+				if err != nil {
+					return err
+				}
+				p["log_policy_drops"] = on
+			}
+			if len(builtins) > 0 {
+				b := map[string]bool{}
+				for _, x := range builtins {
+					k, v, ok := strings.Cut(x, "=")
+					on, err := onOff("builtin "+k, v)
+					if !ok || err != nil {
+						return fmt.Errorf("--builtin %q: write it as NAME=on or NAME=off", x)
+					}
+					b[strings.ReplaceAll(k, "-", "_")] = on
+				}
+				p["builtins"] = b
+			}
+			for flag, v := range map[string][]string{"own-prefix": own, "flowtable": flowtable, "antispoof-v4": spoof4, "antispoof-v6": spoof6} {
+				if fl.Changed(flag) {
+					// "none" (or "") empties the list.
+					if len(v) == 1 && (v[0] == "none" || v[0] == "") {
+						v = []string{}
+					}
+					p[map[string]string{"own-prefix": "own_prefixes", "flowtable": "flowtable_devices", "antispoof-v4": "antispoof_v4", "antispoof-v6": "antispoof_v6"}[flag]] = v
+				}
+			}
+			if fl.Changed("stats-per-device") {
+				on, err := onOff("stats-per-device", perDevice)
+				if err != nil {
+					return err
+				}
+				p["stats_per_device"] = on
+			}
+			if fl.Changed("auto-confirm") {
+				on, err := onOff("auto-confirm", autoConf)
+				if err != nil {
+					return err
+				}
+				p["manual_confirm"] = !on
+			}
+			if fl.Changed("probation") {
+				n, err := parseSeconds(probation)
+				if err != nil {
+					return err
+				}
+				p["probation_seconds"] = n
+			}
+			return change("PATCH", "/settings", p)
+		},
+	}
+	f := set.Flags()
+	f.StringVar(&input, "input-policy", "", "accept or drop what no rule decided, to the firewall")
+	f.StringVar(&forward, "forward-policy", "", "… through it")
+	f.StringVar(&output, "output-policy", "", "… from it")
+	f.StringVar(&logDrops, "log-drops", "", "log what the input and forward policies drop: on or off")
+	f.StringArrayVar(&builtins, "builtin", nil, "turn a built-in on or off: NAME=on|off (repeat)")
+	f.StringArrayVar(&own, "own-prefix", nil, "your public prefixes, for antispoofing (repeat; replaces)")
+	f.StringArrayVar(&flowtable, "flowtable", nil, "fast-path devices (repeat; replaces; none: off)")
+	f.StringArrayVar(&spoof4, "antispoof-v4", nil, "IPv4 sources dropped from the internet (replaces)")
+	f.StringArrayVar(&spoof6, "antispoof-v6", nil, "IPv6 sources dropped from the internet (replaces)")
+	f.StringVar(&probation, "probation", "", "how long a network change has to be confirmed (2m)")
+	f.StringVar(&perDevice, "stats-per-device", "", "count traffic per inside device, for the statistics' top devices: on or off")
+	f.StringVar(&autoConf, "auto-confirm", "", "on: the client that made a change confirms it once it reaches Housecarl again over a new connection; off: a person confirms")
+	cmd.AddCommand(set)
+	return cmd
+}
+
+func onOff(name, v string) (bool, error) {
+	switch strings.ToLower(v) {
+	case "on", "yes", "true":
+		return true, nil
+	case "off", "no", "false":
+		return false, nil
+	}
+	return false, fmt.Errorf("--%s: use on or off", name)
+}
+
+// ---- the ruleset, counters, the log
+
+func rulesetCmd() *cobra.Command {
+	var enforcing bool
+	cmd := &cobra.Command{
+		Use: "ruleset", Short: "Print the nftables ruleset Housecarl builds", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var q url.Values
+			if enforcing {
+				q = url.Values{"enforcing": {"1"}}
+			}
+			var text string
+			if err := c.Do(ctx(), "GET", "/ruleset", q, nil, &text); err != nil {
+				return err
+			}
+			fmt.Print(text)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVar(&enforcing, "enforcing", false, "in shadow mode: the ruleset loaded once it is off")
+	return cmd
+}
+
+func countersCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "counters", Short: "Hits of every rule, NAT rule and built-in", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cs, err := get[api.Counters]("/counters", nil)
+			if err != nil {
+				return err
+			}
+			return output(cs, func(w io.Writer) {
+				fmt.Fprintf(w, "Since %s\n", cs.Since.Local().Format("2006-01-02 15:04"))
+				fmt.Fprintln(w, "MARK\tPACKETS\tBYTES")
+				var marks []string
+				for m := range cs.Counters {
+					marks = append(marks, m)
+				}
+				sort.Strings(marks)
+				for _, m := range marks {
+					c := cs.Counters[m]
+					fmt.Fprintf(w, "%s\t%d\t%d\n", m, c.Packets, c.Bytes)
+				}
+			})
+		},
+	}
+}
+
+func quotasCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "quotas", Aliases: []string{"quota"}, Short: "What the rules' quotas used, and when they reset", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			qs, err := get[[]api.QuotaInfo]("/quotas", nil)
+			if err != nil {
+				return err
+			}
+			var addrs []string
+			for _, q := range qs {
+				for _, a := range q.Addresses {
+					addrs = append(addrs, a.Address)
+				}
+			}
+			names := lookupNames(addrs...)
+			return output(qs, func(w io.Writer) {
+				fmt.Fprintln(w, "RULE\tUSED\tOF\tRESETS\tNEXT RESET")
+				for _, q := range qs {
+					resets := q.Quota.Reset
+					if resets == "" {
+						resets = "by hand"
+					} else if resets == "monthly" && q.Quota.Day > 1 {
+						resets = fmt.Sprintf("monthly (%d)", q.Quota.Day)
+					}
+					next := "-"
+					if q.NextReset != nil {
+						next = q.NextReset.Local().Format("2006-01-02 15:04")
+					}
+					of := humanBytes(q.Quota.Bytes)
+					if q.Quota.PerSource || q.Quota.PerDestination {
+						of += " each"
+					}
+					used := humanBytes(q.Used)
+					if !q.Quota.PerSource && !q.Quota.PerDestination {
+						used += " (" + percent(q.Used, q.Quota.Bytes) + ")"
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Rule, used, of, resets, next)
+					for _, a := range q.Addresses {
+						fmt.Fprintf(w, "  %s\t%s (%s)\t\t\t\n", named(names, a.Address, a.Address), humanBytes(a.Used), percent(a.Used, q.Quota.Bytes))
+					}
+				}
+			})
+		},
+	}
+}
+
+// percent writes used as a share of total: "25%", "<1%" for a little.
+func percent(used, total int64) string {
+	if total <= 0 || used <= 0 {
+		return "0%"
+	}
+	if used*100 < total {
+		return "<1%"
+	}
+	return fmt.Sprintf("%d%%", used*100/total)
+}
+
+func logCmd() *cobra.Command {
+	var follow bool
+	var limit int
+	var search string
+	cmd := &cobra.Command{
+		Use: "log", Short: "Packets the ruleset logged (dropped by a rule or a policy)", Args: cobra.NoArgs,
+		Long: `Shows the packets rules with --log and the policies (settings
+--log-drops) logged, newest last. -f keeps watching.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var last time.Time
+			print := func(es []api.LogEntry) {
+				sort.Slice(es, func(i, j int) bool { return es[i].Time.Before(es[j].Time) })
+				var addrs []string
+				for _, e := range es {
+					addrs = append(addrs, e.Source, e.Dest)
+				}
+				names := lookupNames(addrs...)
+				for _, e := range es {
+					last = e.Time
+					src, dst := e.Source, e.Dest
+					if e.SrcPort != 0 {
+						src = joinPort(src, e.SrcPort)
+					}
+					if e.DestPort != 0 {
+						dst = joinPort(dst, e.DestPort)
+					}
+					src, dst = named(names, e.Source, src), named(names, e.Dest, dst)
+					what := e.Protocol
+					if e.ICMPType != "" {
+						what += " " + e.ICMPType
+					}
+					if e.TCPFlags != "" {
+						what += " " + e.TCPFlags
+					}
+					if e.Shadow && (e.Verdict == "drop" || e.Verdict == "reject") {
+						what += "  (shadow mode: would " + e.Verdict + ", let through)"
+					}
+					fmt.Printf("%s  %-28s %-6s %-6s %s -> %s  %s\n", e.Time.Local().Format("15:04:05"), strings.TrimSpace(e.Prefix), orDash(e.In), orDash(e.Out), src, dst, what)
+				}
+			}
+			for {
+				var es []api.LogEntry
+				query := q("limit", fmt.Sprint(limit), "q", search)
+				if !last.IsZero() {
+					query.Set("after", last.Format(time.RFC3339Nano))
+				}
+				if err := c.Get(ctx(), "/log", query, &es); err != nil {
+					return err
+				}
+				if flagOutput == "json" || flagOutput == "yaml" {
+					return output(es, nil)
+				}
+				print(es)
+				if !follow {
+					if len(es) == 0 {
+						fmt.Fprintln(os.Stderr, "nothing logged yet (log with 'hc rule update ID --log' or 'hc settings set --log-drops on')")
+					}
+					return nil
+				}
+				time.Sleep(2 * time.Second)
+			}
+		},
+	}
+	cmd.Flags().BoolVarP(&follow, "follow", "f", false, "keep watching")
+	cmd.Flags().IntVar(&limit, "limit", 50, "at most this many")
+	cmd.Flags().StringVar(&search, "search", "", "only entries containing this (address, interface, prefix)")
+	return cmd
+}
+
+func joinPort(addr string, port int) string {
+	if strings.Contains(addr, ":") {
+		return fmt.Sprintf("[%s]:%d", addr, port)
+	}
+	return fmt.Sprintf("%s:%d", addr, port)
+}
+
+// ---- probation
+
+func confirmCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "confirm", Short: "Keep the changes on probation (otherwise they roll back)", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var out map[string]int
+			if err := c.Do(ctx(), "POST", "/confirm", nil, nil, &out); err != nil {
+				return err
+			}
+			if out["confirmed"] == 0 {
+				fmt.Println("nothing was waiting to be confirmed")
+			} else if out["confirmed"] == 1 {
+				fmt.Println("confirmed 1 change; it is kept and loaded at boot")
+			} else {
+				fmt.Printf("confirmed %d changes; they are kept and loaded at boot\n", out["confirmed"])
+			}
+			return nil
+		},
+	}
+}
+
+func rollbackCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "rollback", Short: "Undo the changes on probation now", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var out map[string]int
+			if err := c.Do(ctx(), "POST", "/rollback", nil, nil, &out); err != nil {
+				return err
+			}
+			fmt.Printf("rolled back %d changes\n", out["rolled_back"])
+			return nil
+		},
+	}
+}
+
+// ---- import and takeover
+
+func importCmd() *cobra.Command {
+	var replace bool
+	var table, defines, textFile string
+	var zones []string
+	cmd := &cobra.Command{
+		Use:   "import [FILE|-]",
+		Short: "Import an nftables ruleset (nft -j JSON) as Housecarl's configuration",
+		Long: `Turns an existing nftables ruleset into zones, sets, services, rules, port
+forwards and NAT. The input is nft's JSON; on the firewall:
+
+  sudo hc import --defines /etc/nftables.conf -n
+
+Without a FILE, hc runs 'nft -j list ruleset' and 'nft -a list ruleset'
+itself (root on the firewall); from elsewhere, give the JSON as FILE and
+the text with --text. --defines reads the ruleset file's define lines, to
+name the sets as you did (admins, lan_net, …), and the zones; --zone
+enp2s0=lan names one yourself (repeat; interfaces given one name share a
+zone). -n shows the result without
+saving; what can't be expressed becomes raw rules, listed with notes.
+Once it looks right, import without -n, check it with 'hc ruleset', and
+'hc takeover inet main' deletes the old table.`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var data, text []byte
+			var err error
+			switch {
+			case len(args) == 0:
+				data, err = exec.Command("nft", "-j", "list", "ruleset").Output()
+				if err != nil {
+					return fmt.Errorf("running 'nft -j list ruleset': %v (give a FILE, or run as root on the firewall)", err)
+				}
+				text, _ = exec.Command("nft", "-a", "list", "ruleset").Output()
+			case args[0] == "-":
+				data, err = io.ReadAll(os.Stdin)
+			default:
+				data, err = os.ReadFile(args[0])
+			}
+			if err != nil {
+				return err
+			}
+			if textFile != "" {
+				if text, err = os.ReadFile(textFile); err != nil {
+					return err
+				}
+			}
+			in := api.ImportInput{JSON: data, Text: string(text), Table: table, Replace: replace, Zones: map[string]string{}}
+			for _, z := range zones {
+				iface, name, ok := strings.Cut(z, "=")
+				if !ok || iface == "" || name == "" {
+					return fmt.Errorf("--zone %q: write it as INTERFACE=NAME (enp2s0=lan)", z)
+				}
+				in.Zones[iface] = name
+			}
+			if defines != "" {
+				b, err := os.ReadFile(defines)
+				if err != nil {
+					return err
+				}
+				in.Defines = string(b)
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var res api.ImportResult
+			if err := c.Do(ctx(), "POST", "/import/nft", nil, in, &res); err != nil {
+				return err
+			}
+			if flagOutput == "json" || flagOutput == "yaml" {
+				return output(res, nil)
+			}
+			st := res.State
+			fmt.Printf("%d zones, %d sets, %d services, %d rules, %d port forwards, %d source NAT rules, %d redirects\n",
+				len(st.Zones), len(st.Sets), len(st.Services), len(st.Rules), len(st.Forwards), len(st.SNATs), len(st.Redirects))
+			for _, n := range res.Notes {
+				fmt.Println("note:", n)
+			}
+			for _, r := range res.Raw {
+				fmt.Println("raw rule:", r)
+			}
+			return report(&res.MutationResponse)
+		},
+	}
+	cmd.Flags().BoolVar(&replace, "replace", false, "replace Housecarl's configuration (needed when it has one)")
+	cmd.Flags().StringVar(&table, "table", "", "import only this table (\"inet main\"; default: all but Housecarl's)")
+	cmd.Flags().StringVar(&defines, "defines", "", "the ruleset file, whose defines name the sets")
+	cmd.Flags().StringVar(&textFile, "text", "", "'nft -a list ruleset' output, to keep what can't be expressed as raw rules")
+	cmd.Flags().StringArrayVar(&zones, "zone", nil, "name an interface's zone: INTERFACE=NAME (repeat)")
+	return cmd
+}
+
+func takeoverCmd() *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use:   "takeover TABLE...",
+		Short: "Delete the tables Housecarl replaces (\"inet main\"), on probation",
+		Long: `Deletes other nftables tables once Housecarl's ruleset replaces them (after
+'hc import'). Until then both apply: a packet must get past both. Over the
+network this is on probation like a change: unless confirmed, the tables
+come back. Don't forget the boot: disable nftables.service, or empty
+/etc/nftables.conf, or it brings the old ruleset back.
+
+  hc takeover "inet main"`,
+		Args: cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !confirm("Delete "+strings.Join(args, ", ")+"?", yes) {
+				return fmt.Errorf("nothing deleted")
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var out map[string]any
+			if err := c.Do(ctx(), "POST", "/takeover", nil, api.TakeoverInput{Tables: args}, &out); err != nil {
+				return err
+			}
+			fmt.Println(out["message"])
+			if tok, _ := out["confirm_token"].(string); tok != "" && !flagNoConfirm {
+				if err := autoConfirm(tok); err != nil {
+					fmt.Printf("couldn't confirm it (%v): confirm with 'hc confirm', or the tables come back\n", err)
+				} else {
+					fmt.Println("confirmed: this client reached Housecarl again over a new connection")
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	return cmd
+}

cmd/hc/main.go

@@ -1,42 +1,497 @@
-// Command hc is Housecarl's command line client: it will manage the
-// firewall through housecarld's API, as jarl and steward do for theirs.
-// For now it only knows its version.
+// Command hc is the command line client for Housecarl. It talks to
+// housecarld's REST API: on the server itself through the local socket, or
+// anywhere with the server's URL and an API token.
 package main
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
 
 	"github.com/spf13/cobra"
+	"go.yaml.in/yaml/v3"
 
+	"housecarl/internal/api"
+	"housecarl/internal/client"
+	"housecarl/internal/model"
+	"housecarl/internal/validate"
 	"housecarl/internal/version"
 )
 
 func main() {
 	if err := rootCmd().Execute(); err != nil {
-		fmt.Fprintln(os.Stderr, "error:", err)
-		os.Exit(1)
+		printError(err)
+		os.Exit(exitCode(err))
 	}
 }
 
+// Global flags.
+var (
+	flagServer    string
+	flagToken     string
+	flagConfig    string
+	flagDryRun    bool
+	flagOutput    string
+	flagInsecure  bool
+	flagCACert    string
+	flagQuiet     bool
+	flagNoConfirm bool
+	flagNoNames   bool
+	// lastClient is the client the command made (to confirm a change by
+	// itself after it).
+	lastClient *client.Client
+)
+
+// DefaultSocket is housecarld's local socket.
+const DefaultSocket = "/run/housecarl/housecarl.sock"
+
+// cliConfig is the CLI's configuration file.
+type cliConfig struct {
+	Server   string `yaml:"server,omitempty"`
+	Token    string `yaml:"token,omitempty"`
+	Insecure bool   `yaml:"insecure,omitempty"`
+	CACert   string `yaml:"ca_cert,omitempty"`
+}
+
+func defaultConfigPath() string {
+	if p := os.Getenv("HOUSECARL_CLI_CONFIG"); p != "" {
+		return p
+	}
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		dir = filepath.Join(os.Getenv("HOME"), ".config")
+	}
+	return filepath.Join(dir, "housecarl", "config.yaml")
+}
+
+func loadCLIConfig() (*cliConfig, error) {
+	c := &cliConfig{}
+	b, err := os.ReadFile(flagConfig)
+	if errors.Is(err, os.ErrNotExist) {
+		return c, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return c, yaml.Unmarshal(b, c)
+}
+
+func saveCLIConfig(c *cliConfig) error {
+	if err := os.MkdirAll(filepath.Dir(flagConfig), 0o700); err != nil {
+		return err
+	}
+	b, err := yaml.Marshal(c)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(flagConfig, b, 0o600)
+}
+
+func firstNonEmpty(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+// newClient connects to the server: --server, HOUSECARL_SERVER, the config
+// file, or else the local socket.
+func newClient() (*client.Client, error) {
+	cfg, err := loadCLIConfig()
+	if err != nil {
+		return nil, fmt.Errorf("reading %s: %w", flagConfig, err)
+	}
+	server := firstNonEmpty(flagServer, os.Getenv("HOUSECARL_SERVER"), cfg.Server)
+	token := firstNonEmpty(flagToken, os.Getenv("HOUSECARL_TOKEN"), cfg.Token)
+	if server == "" || server == "local" {
+		sock := firstNonEmpty(os.Getenv("HOUSECARL_SOCKET"), DefaultSocket)
+		if _, err := os.Stat(sock); err != nil {
+			return nil, fmt.Errorf("no server configured, and there is no local socket %s: use --server URL, HOUSECARL_SERVER or 'hc config set server https://...'; on the server itself, check that housecarld runs", sock)
+		}
+		server = "unix://" + sock
+	}
+	if strings.HasPrefix(token, "@") {
+		b, err := os.ReadFile(token[1:])
+		if err != nil {
+			return nil, fmt.Errorf("reading the token file: %w", err)
+		}
+		token = strings.TrimSpace(string(b))
+	}
+	if _, isSocket := client.SocketPath(server); !isSocket && token == "" {
+		return nil, errors.New("no API token: use --token, HOUSECARL_TOKEN or 'hc config set token'")
+	}
+	c, err := client.New(client.Options{Server: server, Token: token, Insecure: flagInsecure || cfg.Insecure, CACert: firstNonEmpty(flagCACert, cfg.CACert)})
+	if err != nil {
+		return nil, err
+	}
+	c.DryRun = flagDryRun
+	if err := setupChangeset(c); err != nil {
+		return nil, err
+	}
+	lastClient = c
+	return c, nil
+}
+
 func rootCmd() *cobra.Command {
 	root := &cobra.Command{
 		Use:   "hc",
-		Short: "Manage a Housecarl firewall",
-		Long: `hc will manage a Housecarl firewall through housecarld's API: its zones,
-address sets, rules and port forwards. It doesn't yet: Housecarl is at its
-start (docs/design.md).`,
+		Short: "Command line client for Housecarl, the firewall",
+		Long: `hc manages a Housecarl firewall through its API.
+
+Every change is checked as a whole, then by nftables, before it is put in
+force, and is recorded in the history, where 'hc history undo ID' takes it
+back. --dry-run shows what a change would do, with the ruleset's diff.
+Changes are made at once; to review several and apply them together,
+stage them in a changeset ('hc changeset --help', -C ID or
+HOUSECARL_CHANGESET).
+
+A change made over the network is on probation: hc confirms it by itself
+once it reaches Housecarl again over a new connection, and a change that
+cuts you off can't be confirmed, so it is rolled back by itself (within 2
+minutes by default). --no-confirm leaves it to 'hc confirm'. On the firewall itself, root uses
+housecarld's local socket: no token, no probation. From elsewhere, set the
+server and an API token once:
+
+  hc config set server https://firewall.example.com:8070
+  hc config set token     (paste the token at the prompt)
+
+Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found,
+5 conflict, 6 invalid or rejected by validation, 7 server error.`,
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		Version:       version.String(),
 	}
-	root.AddCommand(versionCmd())
+	pf := root.PersistentFlags()
+	pf.StringVarP(&flagServer, "server", "s", "", "server URL, or \"local\" for the local socket (env HOUSECARL_SERVER)")
+	pf.StringVar(&flagToken, "token", "", "API token, or @FILE to read it from a file (env HOUSECARL_TOKEN)")
+	pf.StringVar(&flagConfig, "config", defaultConfigPath(), "CLI configuration file")
+	pf.BoolVarP(&flagDryRun, "dry-run", "n", false, "check and show a change without making it")
+	pf.StringVarP(&flagOutput, "output", "o", "table", "output format: table, json, yaml or id")
+	pf.BoolVar(&flagInsecure, "insecure", false, "don't verify the server's TLS certificate")
+	pf.StringVar(&flagCACert, "ca-cert", "", "PEM file with a CA to trust for the server's certificate")
+	pf.BoolVarP(&flagQuiet, "quiet", "q", false, "print only errors (and IDs of new objects)")
+	pf.BoolVar(&flagNoConfirm, "no-confirm", false, "leave a change on probation instead of confirming it once this client reaches Housecarl again")
+	pf.BoolVar(&flagNoNames, "no-names", false, "show inside addresses without their names (reverse lookups)")
+	pf.Int64VarP(&flagChangeset, "changeset", "C", 0, "stage configuration changes in this changeset instead of applying them (env HOUSECARL_CHANGESET)")
+	root.AddGroup(&cobra.Group{ID: "fw", Title: "Firewall:"}, &cobra.Group{ID: "changes", Title: "Changes:"}, &cobra.Group{ID: "admin", Title: "Administration:"})
+	add := func(group string, cmds ...*cobra.Command) {
+		for _, c := range cmds {
+			c.GroupID = group
+			root.AddCommand(c)
+		}
+	}
+	add("fw", statusCmd(), zoneCmd(), setCmd(), serviceCmd(), ruleCmd(), forwardCmd(), snatCmd(), redirectCmd(), banlistCmd(), bansCmd(), banCmd(), unbanCmd(), settingsCmd(),
+		rulesetCmd(), countersCmd(), quotasCmd(), logCmd(), connectionsCmd(), traceCmd(), matrixCmd(), statsCmd(), interfaceCmd())
+	add("changes", changesetCmd(), confirmCmd(), rollbackCmd(), historyCmd(), importCmd(), takeoverCmd())
+	add("admin", whoamiCmd(), userCmd(), tokenCmd(), passwordCmd(), certCmd(), auditCmd(), configCmd())
 	return root
 }
 
-func versionCmd() *cobra.Command {
-	return &cobra.Command{
-		Use: "version", Short: "Print the version", Args: cobra.NoArgs,
-		Run: func(cmd *cobra.Command, args []string) { fmt.Println("hc", version.String()) },
+func ctx() context.Context { return context.Background() }
+
+// ---- output
+
+func output(v any, table func(w io.Writer)) error {
+	switch flagOutput {
+	case "json":
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		return enc.Encode(v)
+	case "yaml":
+		b, err := json.Marshal(v)
+		if err != nil {
+			return err
+		}
+		var x any
+		if err := json.Unmarshal(b, &x); err != nil {
+			return err
+		}
+		return yaml.NewEncoder(os.Stdout).Encode(x)
+	case "id":
+		return outputIDs(v)
+	case "table", "":
+		if table == nil {
+			flagOutput = "yaml"
+			return output(v, nil)
+		}
+		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
+		table(tw)
+		return tw.Flush()
+	}
+	return fmt.Errorf("unknown output format %q (use table, json, yaml or id)", flagOutput)
+}
+
+// outputIDs prints the id of each object.
+func outputIDs(v any) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.UseNumber()
+	var x any
+	if err := dec.Decode(&x); err != nil {
+		return err
+	}
+	list, ok := x.([]any)
+	if !ok {
+		list = []any{x}
+	}
+	for _, it := range list {
+		if m, ok := it.(map[string]any); ok {
+			if id, ok := m["id"]; ok {
+				fmt.Println(id)
+			}
+		}
+	}
+	return nil
+}
+
+func printProblems(w io.Writer, ps []validate.Problem) {
+	for _, p := range ps {
+		obj := ""
+		if p.Object != "" {
+			obj = p.Object + ": "
+		}
+		fmt.Fprintf(w, "%s: %s%s [%s]\n", p.Severity, obj, p.Message, p.Code)
 	}
 }
+
+func printError(err error) {
+	var ce *client.Error
+	if errors.As(err, &ce) {
+		ps := ce.Info.Problems
+		if len(ps) == 1 {
+			fmt.Fprintf(os.Stderr, "error: %s [%s]\n", ce.Info.Message, ps[0].Code)
+			return
+		}
+		fmt.Fprintln(os.Stderr, "error:", ce.Info.Message)
+		if len(ps) > 1 {
+			printProblems(os.Stderr, ps)
+		}
+		return
+	}
+	fmt.Fprintln(os.Stderr, "error:", err)
+}
+
+// exitCode maps errors to exit codes: 1 other errors, 3 not authenticated
+// or not allowed, 4 not found, 5 conflict, 6 invalid request or rejected
+// by validation, 7 server error.
+func exitCode(err error) int {
+	var ce *client.Error
+	if errors.As(err, &ce) {
+		switch {
+		case ce.Status == 401 || ce.Status == 403:
+			return 3
+		case ce.Status == 404:
+			return 4
+		case ce.Status == 409:
+			return 5
+		case ce.Status == 400 || ce.Status == 422:
+			return 6
+		case ce.Status >= 500:
+			return 7
+		}
+	}
+	return 1
+}
+
+// autoConfirm confirms a change with its token: the request goes over a
+// new connection (the server closed the change's), so it gets through
+// only if the new ruleset still lets this client reach Housecarl.
+func autoConfirm(token string) error {
+	if lastClient == nil {
+		return errors.New("no client")
+	}
+	c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var out map[string]int
+	return lastClient.Do(c, "POST", "/confirm", nil, api.ConfirmInput{Token: token}, &out)
+}
+
+// report prints the outcome of a change.
+func report(resp *api.MutationResponse) error {
+	switch flagOutput {
+	case "json", "yaml":
+		return output(resp, nil)
+	case "id":
+		if len(resp.Object) > 0 {
+			return outputIDs(resp.Object)
+		}
+		return nil
+	}
+	w := os.Stdout
+	verb := func(c model.Change) string {
+		if resp.DryRun {
+			return "would " + c.Action()
+		}
+		if resp.Staged {
+			return "staged: " + c.Action()
+		}
+		return c.Action() + "d"
+	}
+	if !flagQuiet || resp.DryRun {
+		for _, c := range resp.Changes {
+			if c.Kind == model.KindSettings {
+				fmt.Fprintf(w, "%s settings\n", verb(c))
+				continue
+			}
+			fmt.Fprintf(w, "%s %s %s\n", verb(c), c.Kind, c.Label)
+		}
+	}
+	switch {
+	case resp.Staged && resp.Changeset != nil:
+		if !flagQuiet {
+			cs := resp.Changeset
+			fmt.Fprintf(w, "staged in changeset %d as operation %d (%s now); review with 'hc changeset diff %d', apply with 'hc changeset apply %d'\n",
+				cs.ID, cs.Ops[len(cs.Ops)-1].Seq, plural(len(cs.Ops), "operation"), cs.ID, cs.ID)
+		}
+	case resp.DryRun:
+		if resp.Diff != "" && !flagQuiet {
+			fmt.Fprintln(w)
+			printDiff(w, resp.Diff)
+			fmt.Fprintln(w)
+		}
+		fmt.Fprintln(w, "dry run: nothing was changed")
+	case resp.Applied && resp.ConfirmBy != nil && resp.ConfirmToken != "" && !flagNoConfirm:
+		if err := autoConfirm(resp.ConfirmToken); err != nil {
+			fmt.Fprintf(w, "applied on probation (history %d), but couldn't confirm it: %v\nIt is rolled back within %s unless confirmed ('hc confirm')\n",
+				resp.HistoryID, err, time.Until(*resp.ConfirmBy).Round(time.Second))
+		} else if !flagQuiet {
+			fmt.Fprintf(w, "applied and confirmed (history %d): this client reached Housecarl again over a new connection\n", resp.HistoryID)
+		}
+	case resp.Applied && resp.ConfirmBy != nil:
+		fmt.Fprintf(w, "applied on probation (history %d): confirm with 'hc confirm' within %s, or it is rolled back\n",
+			resp.HistoryID, time.Until(*resp.ConfirmBy).Round(time.Second))
+	case resp.Applied && !flagQuiet:
+		fmt.Fprintf(w, "applied and saved (history %d; undo with 'hc history undo %d')\n", resp.HistoryID, resp.HistoryID)
+	case !resp.Applied && len(resp.Changes) == 0 && !flagQuiet:
+		fmt.Fprintln(w, "nothing to change")
+	}
+	if flagQuiet && len(resp.Object) > 0 && (resp.Applied || resp.Staged) {
+		outputIDs(resp.Object)
+	}
+	printProblems(os.Stderr, resp.Warnings)
+	return nil
+}
+
+func q(kv ...string) url.Values {
+	v := url.Values{}
+	for i := 0; i+1 < len(kv); i += 2 {
+		if kv[i+1] != "" {
+			v.Set(kv[i], kv[i+1])
+		}
+	}
+	return v
+}
+
+func confirm(prompt string, yes bool) bool {
+	if yes || flagDryRun {
+		return true
+	}
+	fmt.Fprintf(os.Stderr, "%s [y/N] ", prompt)
+	var answer string
+	fmt.Scanln(&answer)
+	return strings.EqualFold(answer, "y") || strings.EqualFold(answer, "yes")
+}
+
+// ---- durations
+
+// parseSeconds reads a duration: "12h", "30m", "7d", "1d12h" or plain
+// seconds.
+func parseSeconds(s string) (int, error) {
+	s = strings.TrimSpace(s)
+	if n, err := strconv.Atoi(s); err == nil {
+		return n, nil
+	}
+	days := 0
+	if i := strings.Index(s, "d"); i > 0 {
+		n, err := strconv.Atoi(s[:i])
+		if err != nil {
+			return 0, fmt.Errorf("invalid duration %q (use e.g. 12h, 30m or 7d)", s)
+		}
+		days, s = n, s[i+1:]
+	}
+	var d time.Duration
+	if s != "" {
+		var err error
+		if d, err = time.ParseDuration(s); err != nil {
+			return 0, fmt.Errorf("invalid duration %q (use e.g. 12h, 30m or 7d)", s)
+		}
+	}
+	return days*86400 + int(d/time.Second), nil
+}
+
+// fmtSeconds writes seconds the short way: 43200 is "12h", 90 is "1m30s".
+func fmtSeconds(n int) string {
+	if n <= 0 {
+		return "-"
+	}
+	var parts []string
+	for _, u := range []struct {
+		s    int
+		name string
+	}{{86400, "d"}, {3600, "h"}, {60, "m"}, {1, "s"}} {
+		if n >= u.s {
+			parts = append(parts, strconv.Itoa(n/u.s)+u.name)
+			n %= u.s
+		}
+	}
+	return strings.Join(parts, "")
+}
+
+// fmtWhen writes a time relative to now: "in 3h", "2d ago".
+func fmtWhen(t time.Time) string {
+	if t.IsZero() {
+		return "-"
+	}
+	d := time.Until(t)
+	ago := d < 0
+	if ago {
+		d = -d
+	}
+	var s string
+	switch {
+	case d < time.Minute:
+		return "now"
+	case d < time.Hour:
+		s = fmt.Sprintf("%dm", int(d.Minutes()))
+	case d < 48*time.Hour:
+		s = fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
+	default:
+		s = fmt.Sprintf("%dd", int(d.Hours()/24))
+	}
+	if ago {
+		return s + " ago"
+	}
+	return "in " + s
+}
+
+// fmtSeen is a last-seen time, or "-" for a client never seen.
+func fmtSeen(t *time.Time) string {
+	if t == nil {
+		return "-"
+	}
+	return fmtWhen(*t)
+}
+
+func orDash(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}
+
+// optionsArg turns --option NAME=VALUE flags into option inputs.

cmd/hc/main_test.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+
+	"housecarl/internal/matrix"
+	"housecarl/internal/model"
+)
+
+func TestDurations(t *testing.T) {
+	for in, want := range map[string]int{"3600": 3600, "12h": 43200, "30m": 1800, "1d": 86400, "1d12h": 129600, "1h30m": 5400} {
+		if got, err := parseSeconds(in); err != nil || got != want {
+			t.Errorf("parseSeconds(%q) = %d, %v; want %d", in, got, err, want)
+		}
+	}
+	for _, bad := range []string{"12x", "d", "1d2x"} {
+		if _, err := parseSeconds(bad); err == nil {
+			t.Errorf("parseSeconds(%q) accepted", bad)
+		}
+	}
+	for n, want := range map[int]string{43200: "12h", 5400: "1h30m", 90061: "1d1h1m1s", 0: "-"} {
+		if got := fmtSeconds(n); got != want {
+			t.Errorf("fmtSeconds(%d) = %q, want %q", n, got, want)
+		}
+	}
+}
+
+func TestLimitAndDescribe(t *testing.T) {
+	l, err := parseLimit("3/minutes,burst=5,over,per-source")
+	if err != nil || l.Rate != 3 || l.Per != "minute" || l.Burst != 5 || !l.Over || !l.PerSource {
+		t.Errorf("parseLimit: %+v %v", l, err)
+	}
+	if _, err := parseLimit("fast"); err == nil {
+		t.Error("a bad limit accepted")
+	}
+	for _, c := range []struct {
+		m     model.Match
+		chain string
+		want  string
+	}{
+		{model.Match{InZones: []string{"wan"}, NotSources: true, Sources: []string{"trusted"}, Services: []string{"ssh", "tcp/8080"}}, "input",
+			"wan → firewall  from any but trusted  ssh,tcp/8080"},
+		{model.Match{InZones: []string{"wan"}, NotIn: true, Destinations: []string{"trusted_ips"}}, "forward", "any but wan → any  to trusted_ips"},
+		{model.Match{InZones: []string{"lan"}, OutZones: []string{"wan"}}, "forward", "lan → wan"},
+		{model.Match{OutZones: []string{"wan"}}, "output", "firewall → wan"},
+		{model.Match{OutZones: []string{"wan"}, Sources: []string{"10.0.0.0/8"}}, "", "any → wan  from 10.0.0.0/8"},
+		{model.Match{Sources: []string{"trusted"}}, "input", "from trusted"},
+		{model.Match{}, "forward", "everything"},
+	} {
+		if got := describe(c.m, c.chain); got != c.want {
+			t.Errorf("describe(%+v, %s) = %q, want %q", c.m, c.chain, got, c.want)
+		}
+	}
+	for used, want := range map[int64]string{0: "0%", 5: "<1%", 250: "25%", 1000: "100%", 1300: "130%"} {
+		if got := percent(used, 1000); got != want {
+			t.Errorf("percent(%d, 1000) = %q, want %q", used, got, want)
+		}
+	}
+	if got := editList([]string{"a", "b", "c"}, []string{"d"}, []string{"b"}); len(got) != 3 || got[1] != "c" || got[2] != "d" {
+		t.Errorf("editList: %v", got)
+	}
+}
+
+// TestMatrixAligned: the grid's columns line up in colour too (the
+// escape codes take no room on screen).
+func TestMatrixAligned(t *testing.T) {
+	st := model.NewState()
+	st.Zones = []*model.Zone{{ID: 1, Name: "lan", Interfaces: []string{"eth1"}}, {ID: 2, Name: "wan", Interfaces: []string{"wan"}, External: true}}
+	st.Rules = []*model.Rule{{ID: 3, Chain: model.ChainForward, Position: 1, Action: model.ActionAccept, Match: model.Match{InZones: []string{"lan"}, Services: []string{"https"}}}}
+	st.Settings.InputPolicy, st.Settings.ForwardPolicy = model.PolicyDrop, model.PolicyDrop
+	var b strings.Builder
+	writeMatrix(&b, matrix.Build(st), true)
+	esc := regexp.MustCompile("\x1b\\[[0-9]+m")
+	lines := strings.Split(esc.ReplaceAllString(b.String(), ""), "\n")
+	col := strings.Index(lines[0], "wan")
+	for _, l := range lines[1:4] {
+		if r := []rune(l); len(r) <= col || r[col-1] != ' ' || r[col] == ' ' {
+			t.Errorf("misaligned at column %d:\n%s", col, strings.Join(lines[:4], "\n"))
+			break
+		}
+	}
+	if !strings.Contains(b.String(), "\x1b[33mpartly +1") || !strings.Contains(b.String(), "lan → wan\thttps") {
+		t.Errorf("no partly cell, or no list of what's open:\n%s", b.String())
+	}
+}

cmd/hc/objects.go

@@ -0,0 +1,632 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+// ---- helpers shared by the object commands
+
+// change sends a change and reports it.
+func change(method, path string, body any) error {
+	c, err := newClient()
+	if err != nil {
+		return err
+	}
+	var resp api.MutationResponse
+	if err := c.Do(ctx(), method, path, nil, body, &resp); err != nil {
+		return err
+	}
+	return report(&resp)
+}
+
+func objPath(kind, ref string) string { return "/" + kind + "/" + url.PathEscape(ref) }
+
+func get[T any](path string, query url.Values) (T, error) {
+	var out T
+	c, err := newClient()
+	if err != nil {
+		return out, err
+	}
+	err = c.Get(ctx(), path, query, &out)
+	return out, err
+}
+
+// counters returns the hit counters, or none when they can't be read
+// (no log:read permission).
+func counters() map[string]api.Counter {
+	cs, err := get[api.Counters]("/counters", nil)
+	if err != nil {
+		return map[string]api.Counter{}
+	}
+	return cs.Counters
+}
+
+// lookupNames asks for inside addresses' names, for tables: none with
+// --no-names, another output format, or when they can't be had.
+func lookupNames(addrs ...string) map[string]string {
+	out := map[string]string{}
+	if flagNoNames || flagOutput != "table" {
+		return out
+	}
+	seen := map[string]bool{}
+	var list []string
+	for _, a := range addrs {
+		if a != "" && !seen[a] {
+			seen[a] = true
+			list = append(list, a)
+		}
+	}
+	if len(list) == 0 {
+		return out
+	}
+	c, err := newClient()
+	if err != nil {
+		return out
+	}
+	var ns api.Names
+	if c.Do(ctx(), "POST", "/names", nil, api.NamesInput{Addresses: list}, &ns) != nil {
+		return out
+	}
+	for a, n := range ns.Names {
+		out[a] = strings.SplitN(n, ".", 2)[0]
+	}
+	return out
+}
+
+// named adds an address's name to text (the address, or address:port).
+func named(names map[string]string, addr, text string) string {
+	if n := names[addr]; n != "" {
+		return text + " (" + n + ")"
+	}
+	return text
+}
+
+func hits(c map[string]api.Counter, marks ...string) string {
+	var p uint64
+	found := false
+	for _, m := range marks {
+		if x, ok := c[m]; ok {
+			p += x.Packets
+			found = true
+		}
+	}
+	if !found {
+		return "-"
+	}
+	return humanCount(p)
+}
+
+func humanCount(n uint64) string {
+	switch {
+	case n >= 10_000_000_000:
+		return fmt.Sprintf("%dG", n/1_000_000_000)
+	case n >= 10_000_000:
+		return fmt.Sprintf("%dM", n/1_000_000)
+	case n >= 10_000:
+		return fmt.Sprintf("%dk", n/1_000)
+	}
+	return fmt.Sprint(n)
+}
+
+func list(vals []string) string { return orDash(strings.Join(vals, ", ")) }
+
+// patch collects the flags given on an update command.
+type patch map[string]any
+
+func deleteCmd(kind, what string) *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use: "delete " + strings.ToUpper(what), Short: "Delete a " + what, Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !confirm(fmt.Sprintf("Delete %s %s?", what, args[0]), yes) {
+				return fmt.Errorf("not deleted")
+			}
+			return change("DELETE", objPath(kind, args[0]), nil)
+		},
+	}
+	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	return cmd
+}
+
+func showCmd[T any](kind, what string, print func(w io.Writer, o T)) *cobra.Command {
+	return &cobra.Command{
+		Use: "show " + strings.ToUpper(what), Short: "Show a " + what, Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			o, err := get[T](objPath(kind, args[0]), nil)
+			if err != nil {
+				return err
+			}
+			return output(o, func(w io.Writer) { print(w, o) })
+		},
+	}
+}
+
+// ---- zones
+
+func zoneCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "zone", Aliases: []string{"zones"}, Short: "Zones: groups of interfaces rules refer to"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the zones", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			zs, err := get[[]model.Zone]("/zones", nil)
+			if err != nil {
+				return err
+			}
+			return output(zs, func(w io.Writer) {
+				fmt.Fprintln(w, "NAME\tINTERFACES\tEXTERNAL\tCOMMENT")
+				for _, z := range zs {
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", z.Name, list(z.Interfaces), yesNo(z.External), z.Comment)
+				}
+			})
+		},
+	})
+	cmd.AddCommand(showCmd("zones", "zone", func(w io.Writer, z model.Zone) {
+		fmt.Fprintf(w, "Name:\t%s\nInterfaces:\t%s\nExternal:\t%s\n", z.Name, list(z.Interfaces), yesNo(z.External))
+		if z.Comment != "" {
+			fmt.Fprintf(w, "Comment:\t%s\n", z.Comment)
+		}
+	}))
+	var ifaces []string
+	var external bool
+	var comment string
+	add := &cobra.Command{
+		Use: "add NAME --interface IFACE...", Short: "Add a zone", Args: cobra.ExactArgs(1),
+		Long: `Adds a zone: interfaces rules can then name together. Wildcards match
+many: --interface 'incusbr*'. --external marks a zone facing the internet:
+antispoofing, MSS clamping and port forwards apply there.
+
+  hc zone add wan --interface wan --external
+  hc zone add lan --interface eth1`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return change("POST", "/zones", model.Zone{Name: args[0], Interfaces: ifaces, External: external, Comment: comment})
+		},
+	}
+	add.Flags().StringArrayVarP(&ifaces, "interface", "i", nil, "an interface (repeat; a trailing * matches many)")
+	add.Flags().BoolVar(&external, "external", false, "the zone faces the internet")
+	add.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	var addIf, removeIf []string
+	var name string
+	update := &cobra.Command{
+		Use: "update ZONE", Short: "Change a zone", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{}
+			fl := cmd.Flags()
+			if fl.Changed("interface") || len(addIf) > 0 || len(removeIf) > 0 {
+				cur := ifaces
+				if !fl.Changed("interface") {
+					z, err := get[model.Zone](objPath("zones", args[0]), nil)
+					if err != nil {
+						return err
+					}
+					cur = z.Interfaces
+				}
+				p["interfaces"] = editList(cur, addIf, removeIf)
+			}
+			if fl.Changed("external") {
+				p["external"] = external
+			}
+			if fl.Changed("comment") {
+				p["comment"] = comment
+			}
+			if fl.Changed("name") {
+				p["name"] = name
+			}
+			return change("PATCH", objPath("zones", args[0]), p)
+		},
+	}
+	f := update.Flags()
+	f.StringArrayVarP(&ifaces, "interface", "i", nil, "replace the interfaces (repeat)")
+	f.StringArrayVar(&addIf, "add", nil, "add an interface")
+	f.StringArrayVar(&removeIf, "remove", nil, "remove an interface")
+	f.BoolVar(&external, "external", false, "the zone faces the internet (--external=false: it doesn't)")
+	f.StringVar(&comment, "comment", "", "a comment")
+	f.StringVar(&name, "name", "", "a new name")
+	cmd.AddCommand(update)
+	cmd.AddCommand(deleteCmd("zones", "zone"))
+	return cmd
+}
+
+func editList(cur, add, remove []string) []string {
+	out := []string{}
+	drop := map[string]bool{}
+	for _, r := range remove {
+		drop[r] = true
+	}
+	for _, x := range cur {
+		if !drop[x] {
+			out = append(out, x)
+		}
+	}
+	return append(out, add...)
+}
+
+func yesNo(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "no"
+}
+
+// ---- sets
+
+func setCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "set", Aliases: []string{"sets"}, Short: "Address sets: named lists of IPv4 and IPv6 addresses"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the sets", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ss, err := get[[]model.AddressSet]("/sets", nil)
+			if err != nil {
+				return err
+			}
+			return output(ss, func(w io.Writer) {
+				fmt.Fprintln(w, "NAME\tENTRIES\tCOMMENT")
+				for _, s := range ss {
+					entries := list(s.Entries)
+					if s.Jarl != nil {
+						entries = "from Jarl: " + strings.Join(jarlNames(s.Jarl), ", ")
+					}
+					if s.Dynamic {
+						entries = "dynamic"
+						if s.Timeout > 0 {
+							entries += ", timeout " + fmtSeconds(s.Timeout)
+						}
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, entries, s.Comment)
+				}
+			})
+		},
+	})
+	cmd.AddCommand(showCmd("sets", "set", func(w io.Writer, s model.AddressSet) {
+		fmt.Fprintf(w, "Name:\t%s\n", s.Name)
+		if s.Dynamic {
+			fmt.Fprintf(w, "Dynamic:\tyes (entries: hc set entries %s)\n", s.Name)
+			if s.Timeout > 0 {
+				fmt.Fprintf(w, "Timeout:\t%s\n", fmtSeconds(s.Timeout))
+			}
+		}
+		if s.Jarl != nil {
+			fmt.Fprintf(w, "From Jarl:\t%s\n", strings.Join(jarlNames(s.Jarl), ", "))
+			fmt.Fprintf(w, "Within:\t%s\n", strings.Join(s.Jarl.Within, ", "))
+			fmt.Fprintf(w, "Addresses:\thc set jarl %s\n", s.Name)
+		}
+		for i, e := range s.Entries {
+			label := ""
+			if i == 0 {
+				label = "Entries:"
+			}
+			fmt.Fprintf(w, "%s\t%s\n", label, e)
+		}
+		if s.Comment != "" {
+			fmt.Fprintf(w, "Comment:\t%s\n", s.Comment)
+		}
+	}))
+	var dynamic bool
+	var timeout, comment string
+	var jarlN, within []string
+	add := &cobra.Command{
+		Use: "add NAME [ENTRY...]", Short: "Add a set", Args: cobra.MinimumNArgs(1),
+		Long: `Adds a set of addresses, prefixes and ranges, IPv4 and IPv6 mixed:
+
+  hc set add trusted 192.168.1.0/24 192.168.2.0/24 2001:db8:1::/64
+  hc set add guests --dynamic --timeout 24h
+  hc set add nas --jarl nas.example.com@internal --within lan
+
+A dynamic set takes entries at run time, each with its own timeout ('hc
+set entries', or the API with a token for that set alone: a captive
+portal's signed-in clients).
+
+A Jarl set holds the addresses of names in Jarl (their A and AAAA
+records, CNAMEs followed), read again every minute. --jarl takes
+NAME@VIEW, or NAME when only one view has it; --within says where its
+addresses may be (zones: their connected networks; or networks), and the
+others are left out ('hc set jarl NAME' shows them). docs/jarl.md.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s := model.AddressSet{Name: args[0], Entries: args[1:], Dynamic: dynamic, Comment: comment}
+			if len(jarlN) > 0 || len(within) > 0 {
+				s.Jarl = jarlSource(jarlN, within)
+			}
+			if timeout != "" {
+				n, err := parseSeconds(timeout)
+				if err != nil {
+					return err
+				}
+				s.Timeout = n
+			}
+			return change("POST", "/sets", s)
+		},
+	}
+	add.Flags().BoolVar(&dynamic, "dynamic", false, "entries are added at run time")
+	add.Flags().StringVar(&timeout, "timeout", "", "a dynamic set's default entry lifetime (1h, 7d)")
+	add.Flags().StringArrayVar(&jarlN, "jarl", nil, "a name in Jarl whose addresses the set holds: NAME or NAME@VIEW (repeat)")
+	add.Flags().StringArrayVar(&within, "within", nil, "a Jarl set: a zone or network its addresses must lie in (repeat)")
+	add.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	var addE, removeE, entries []string
+	var name string
+	update := &cobra.Command{
+		Use: "update SET", Short: "Change a set", Args: cobra.ExactArgs(1),
+		Long: `Changes a set: --add and --remove change entries, --entry replaces them all.
+
+  hc set update trusted --add 10.65.65.0/24 --remove 10.1.1.0/24`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{}
+			fl := cmd.Flags()
+			if fl.Changed("entry") || len(addE) > 0 || len(removeE) > 0 {
+				cur := entries
+				if !fl.Changed("entry") {
+					s, err := get[model.AddressSet](objPath("sets", args[0]), nil)
+					if err != nil {
+						return err
+					}
+					cur = s.Entries
+				}
+				p["entries"] = editList(cur, addE, removeE)
+			}
+			if fl.Changed("jarl") || fl.Changed("within") {
+				s, err := get[model.AddressSet](objPath("sets", args[0]), nil)
+				if err != nil {
+					return err
+				}
+				j := s.Jarl
+				if j == nil {
+					j = &model.JarlSource{}
+				}
+				if fl.Changed("jarl") {
+					j.Names = jarlSource(jarlN, nil).Names
+				}
+				if fl.Changed("within") {
+					j.Within = within
+				}
+				p["jarl"] = j
+			}
+			if fl.Changed("timeout") {
+				n, err := parseSeconds(timeout)
+				if err != nil {
+					return err
+				}
+				p["timeout"] = n
+			}
+			if fl.Changed("comment") {
+				p["comment"] = comment
+			}
+			if fl.Changed("name") {
+				p["name"] = name
+			}
+			return change("PATCH", objPath("sets", args[0]), p)
+		},
+	}
+	f := update.Flags()
+	f.StringArrayVar(&entries, "entry", nil, "replace the entries (repeat)")
+	f.StringArrayVar(&addE, "add", nil, "add an entry")
+	f.StringArrayVar(&removeE, "remove", nil, "remove an entry")
+	f.StringVar(&timeout, "timeout", "", "a dynamic set's default entry lifetime")
+	f.StringArrayVar(&jarlN, "jarl", nil, "replace a Jarl set's names: NAME or NAME@VIEW (repeat)")
+	f.StringArrayVar(&within, "within", nil, "replace where a Jarl set's addresses may be: zones, networks (repeat)")
+	f.StringVar(&comment, "comment", "", "a comment")
+	f.StringVar(&name, "name", "", "a new name")
+	cmd.AddCommand(update)
+	cmd.AddCommand(deleteCmd("sets", "set"))
+	cmd.AddCommand(entriesCmd())
+	cmd.AddCommand(jarlCmd())
+	return cmd
+}
+
+func jarlSource(names, within []string) *model.JarlSource {
+	j := &model.JarlSource{Names: []model.JarlName{}, Within: within}
+	for _, n := range names {
+		j.Names = append(j.Names, model.ParseJarlName(n))
+	}
+	return j
+}
+
+func jarlNames(j *model.JarlSource) []string {
+	var out []string
+	for _, n := range j.Names {
+		out = append(out, n.String())
+	}
+	return out
+}
+
+// jarlCmd shows what a Jarl set holds, name by name.
+func jarlCmd() *cobra.Command {
+	var refresh bool
+	cmd := &cobra.Command{Use: "jarl SET", Short: "What a Jarl set holds: each name's addresses in Jarl, and which are used", Args: cobra.ExactArgs(1),
+		Long: `Shows a Jarl set's names with the addresses Jarl gives them (CNAMEs
+followed), whether each is used or why it is left out (not within the
+set's zones and networks), and when Jarl last answered. --refresh reads
+Jarl first.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var js api.JarlSet
+			if refresh {
+				err = c.Do(ctx(), "POST", objPath("sets", args[0])+"/jarl/refresh", nil, nil, &js)
+			} else {
+				err = c.Do(ctx(), "GET", objPath("sets", args[0])+"/jarl", nil, nil, &js)
+			}
+			if err != nil {
+				return err
+			}
+			return output(js, func(w io.Writer) {
+				if !js.Configured {
+					fmt.Fprintln(w, "Jarl isn't configured (jarl.server in housecarl.yaml): the set keeps the addresses it last had.")
+				}
+				if js.Error != "" {
+					fmt.Fprintf(w, "Reading Jarl failed: %s (the set keeps the addresses it had)\n", js.Error)
+				}
+				fmt.Fprintf(w, "Set:\t%s\n", js.Set)
+				fmt.Fprintf(w, "Within:\t%s\n", strings.Join(js.Within, ", "))
+				fmt.Fprintf(w, "Holds:\t%s\n", list(js.Entries))
+				if js.Read != nil {
+					fmt.Fprintf(w, "Jarl read:\t%s\n", js.Read.Local().Format("2006-01-02 15:04:05"))
+				}
+				fmt.Fprintln(w)
+				fmt.Fprintln(w, "NAME\tVIEW\tADDRESS\tUSED")
+				for _, n := range js.Names {
+					label := n.Name
+					if len(n.Chain) > 0 {
+						label += " → " + strings.Join(n.Chain, " → ")
+					}
+					view := n.View
+					if view == "" {
+						view = n.Found
+					}
+					if view == "" {
+						view = "-"
+					}
+					switch {
+					case n.Problem != "":
+						fmt.Fprintf(w, "%s\t%s\t-\tno: %s\n", label, view, n.Problem)
+					case len(n.Addresses) == 0 && n.Error != "":
+						fmt.Fprintf(w, "%s\t%s\t-\tnot read: %s\n", label, view, n.Error)
+					case len(n.Addresses) == 0:
+						fmt.Fprintf(w, "%s\t%s\t-\tnot read yet\n", label, view)
+					}
+					for _, a := range n.Addresses {
+						used := "yes"
+						if !a.Used {
+							used = "no: " + a.Why
+						}
+						fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", label, view, a.Address, used)
+					}
+					if n.Error != "" && len(n.Addresses) > 0 {
+						fmt.Fprintf(w, "%s\t%s\t\t(the last read failed: %s; these are from before)\n", label, view, n.Error)
+					}
+				}
+			})
+		}}
+	cmd.Flags().BoolVar(&refresh, "refresh", false, "read Jarl now")
+	return cmd
+}
+
+// entriesCmd manages a dynamic set's run-time entries.
+func entriesCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "entries SET", Short: "A dynamic set's entries: list, add, remove", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			es, err := get[[]api.Element](objPath("sets", args[0])+"/elements", nil)
+			if err != nil {
+				return err
+			}
+			return output(es, func(w io.Writer) {
+				fmt.Fprintln(w, "ADDRESS\tEXPIRES IN")
+				for _, e := range es {
+					exp := "-"
+					if e.Timeout > 0 {
+						exp = fmtSeconds(e.Timeout)
+					}
+					fmt.Fprintf(w, "%s\t%s\n", e.Address, exp)
+				}
+			})
+		}}
+	var timeout string
+	add := &cobra.Command{Use: "add SET ADDRESS", Short: "Add an entry to a dynamic set", Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := api.Element{Address: args[1]}
+			if timeout != "" {
+				n, err := parseSeconds(timeout)
+				if err != nil {
+					return err
+				}
+				e.Timeout = n
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			return c.Do(ctx(), "POST", objPath("sets", args[0])+"/elements", nil, e, nil)
+		}}
+	add.Flags().StringVar(&timeout, "timeout", "", "how long it stays (1h; default: the set's)")
+	cmd.AddCommand(add)
+	cmd.AddCommand(&cobra.Command{Use: "remove SET ADDRESS", Short: "Remove an entry from a dynamic set", Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			return c.Do(ctx(), "DELETE", objPath("sets", args[0])+"/elements/"+url.PathEscape(args[1]), nil, nil, nil)
+		}})
+	return cmd
+}
+
+// ---- services
+
+func serviceCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "service", Aliases: []string{"services"}, Short: "Services: named ports and ICMP types (many are built in)"}
+	var own bool
+	ls := &cobra.Command{
+		Use: "list", Short: "List the services, built-in ones too", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ss, err := get[[]model.Service]("/services", q("own", map[bool]string{true: "1"}[own]))
+			if err != nil {
+				return err
+			}
+			return output(ss, func(w io.Writer) {
+				fmt.Fprintln(w, "NAME\tENTRIES\tSOURCE\tCOMMENT")
+				for _, s := range ss {
+					src := "yours"
+					if s.Builtin {
+						src = "built in"
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", s.Name, list(s.Entries), src, s.Comment)
+				}
+			})
+		},
+	}
+	ls.Flags().BoolVar(&own, "own", false, "only yours")
+	cmd.AddCommand(ls)
+	cmd.AddCommand(showCmd("services", "service", func(w io.Writer, s model.Service) {
+		fmt.Fprintf(w, "Name:\t%s\nEntries:\t%s\n", s.Name, list(s.Entries))
+		if s.Builtin {
+			fmt.Fprintln(w, "Source:\tbuilt in (add one of the same name to change it)")
+		}
+	}))
+	var comment string
+	add := &cobra.Command{
+		Use: "add NAME ENTRY...", Short: "Add a service", Args: cobra.MinimumNArgs(2),
+		Long: `Adds a service: ports of a protocol, and ICMP types.
+
+  hc service add web tcp/8080 tcp/8443
+  hc service add games udp/27015-27030 tcp/27015
+  hc service add pmtu icmp/destination-unreachable icmpv6/packet-too-big`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return change("POST", "/services", model.Service{Name: args[0], Entries: args[1:], Comment: comment})
+		},
+	}
+	add.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	var addE, removeE []string
+	update := &cobra.Command{
+		Use: "update SERVICE", Short: "Change a service", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{}
+			if len(addE) > 0 || len(removeE) > 0 {
+				s, err := get[model.Service](objPath("services", args[0]), nil)
+				if err != nil {
+					return err
+				}
+				p["entries"] = editList(s.Entries, addE, removeE)
+			}
+			if cmd.Flags().Changed("comment") {
+				p["comment"] = comment
+			}
+			return change("PATCH", objPath("services", args[0]), p)
+		},
+	}
+	update.Flags().StringArrayVar(&addE, "add", nil, "add an entry")
+	update.Flags().StringArrayVar(&removeE, "remove", nil, "remove an entry")
+	update.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(update)
+	cmd.AddCommand(deleteCmd("services", "service"))
+	return cmd
+}

cmd/hc/rules.go

@@ -0,0 +1,834 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+// matchFlags are the flags of what a rule, source NAT or redirect
+// matches.
+type matchFlags struct {
+	in, out, from, to, services   []string
+	notIn, notOut, notFrom, notTo bool
+	family                        string
+}
+
+func (m *matchFlags) register(f *pflag.FlagSet, in, out bool) {
+	if in {
+		f.StringArrayVar(&m.in, "in", nil, "coming in on a zone (repeat)")
+		f.BoolVar(&m.notIn, "not-in", false, "on any zone but the --in ones")
+	}
+	if out {
+		f.StringArrayVar(&m.out, "out", nil, "going out on a zone (repeat)")
+		f.BoolVar(&m.notOut, "not-out", false, "on any zone but the --out ones")
+	}
+	f.StringArrayVar(&m.from, "from", nil, "source: a set, address, prefix or range (repeat)")
+	f.BoolVar(&m.notFrom, "not-from", false, "any source but the --from ones")
+	f.StringArrayVar(&m.to, "to", nil, "destination: a set, address, prefix or range (repeat)")
+	f.BoolVar(&m.notTo, "not-to", false, "any destination but the --to ones")
+	f.StringArrayVar(&m.services, "service", nil, "a service (ssh, https, yours) or entry (tcp/8080, udp/1000-2000, icmp/echo-request); repeat")
+	f.StringVar(&m.family, "family", "", "ipv4 or ipv6 only")
+}
+
+func (m *matchFlags) match() model.Match {
+	return model.Match{InZones: m.in, NotIn: m.notIn, OutZones: m.out, NotOut: m.notOut, Sources: m.from, NotSources: m.notFrom,
+		Destinations: m.to, NotDests: m.notTo, Services: m.services, Family: m.family}
+}
+
+// patch adds the given match flags to an update.
+func (m *matchFlags) patch(fl *pflag.FlagSet, p patch) {
+	set := func(flag, field string, v any) {
+		if fl.Lookup(flag) != nil && fl.Changed(flag) {
+			p[field] = v
+		}
+	}
+	set("in", "in_zones", m.in)
+	set("not-in", "not_in", m.notIn)
+	set("out", "out_zones", m.out)
+	set("not-out", "not_out", m.notOut)
+	set("from", "sources", m.from)
+	set("not-from", "not_sources", m.notFrom)
+	set("to", "destinations", m.to)
+	set("not-to", "not_destinations", m.notTo)
+	set("service", "services", m.services)
+	set("family", "family", m.family)
+}
+
+// describe sums up a match in a chain ("" for NAT): "lan → firewall
+// from trusted  ssh,dns". Zones are a direction; the firewall itself is
+// the other end of the input and output chains.
+func describe(m model.Match, chain string) string {
+	var parts []string
+	list := func(not bool, xs []string) string {
+		s := strings.Join(xs, ",")
+		if not {
+			return "any but " + s
+		}
+		return s
+	}
+	if len(m.InZones) > 0 || len(m.OutZones) > 0 {
+		in, out := "any", "any"
+		if len(m.InZones) > 0 {
+			in = list(m.NotIn, m.InZones)
+		}
+		if len(m.OutZones) > 0 {
+			out = list(m.NotOut, m.OutZones)
+		}
+		switch chain {
+		case model.ChainInput:
+			out = "firewall"
+		case model.ChainOutput:
+			in = "firewall"
+		}
+		parts = append(parts, in+" → "+out)
+	}
+	if len(m.Sources) > 0 {
+		parts = append(parts, "from "+list(m.NotSources, m.Sources))
+	}
+	if len(m.Destinations) > 0 {
+		parts = append(parts, "to "+list(m.NotDests, m.Destinations))
+	}
+	if len(m.Services) > 0 {
+		parts = append(parts, strings.Join(m.Services, ","))
+	}
+	if m.Family != "" {
+		parts = append(parts, m.Family+" only")
+	}
+	if len(parts) == 0 {
+		return "everything"
+	}
+	return strings.Join(parts, "  ")
+}
+
+// ---- rules
+
+type ruleFlags struct {
+	matchFlags
+	action, rejectWith, comment, logPrefix, raw string
+	states, statuses, days                      []string
+	hours, limit, quota                         string
+	connLimit, groupV4, groupV6                 int
+	log, disabled, everyPacket                  bool
+	before, after                               int64
+	first                                       bool
+}
+
+func (r *ruleFlags) register(f *pflag.FlagSet) {
+	r.matchFlags.register(f, true, true)
+	f.StringVarP(&r.action, "action", "a", "accept", "accept, drop, reject, or log (log and go on)")
+	f.StringVar(&r.rejectWith, "reject-with", "", "tcp-reset (default: an ICMP error)")
+	f.StringSliceVar(&r.states, "state", nil, "connection states: new, established, related, invalid, untracked")
+	f.StringSliceVar(&r.statuses, "status", nil, "connection status: dnat, snat")
+	f.StringSliceVar(&r.days, "days", nil, "days it applies: Monday,Tuesday,…")
+	f.StringVar(&r.hours, "hours", "", "hours it applies: 08:00-17:30 (22:00-06:00 passes midnight)")
+	f.StringVar(&r.limit, "limit", "", "a rate: 10/minute, or bytes: 1mbytes/second, 500kbytes/minute; with ,burst=N ,over (beyond the rate) ,per-source or ,per-destination")
+	f.IntVar(&r.connLimit, "conn-limit", 0, "match a source with more than this many connections")
+	f.StringVar(&r.quota, "quota", "", "an allowance: 5000mbytes or 5gbytes, with ,over (match once used up) ,per-source or ,per-destination, and ,daily ,weekly or ,monthly=DAY to reset")
+	f.IntVar(&r.groupV4, "group-v4", 0, "per-source/destination limits and quotas count IPv4 addresses by this prefix (default 32: each address)")
+	f.IntVar(&r.groupV6, "group-v6", 0, "…and IPv6 addresses by this prefix (default 64: one device or network)")
+	f.BoolVar(&r.log, "log", false, "log matching packets ('hc log')")
+	f.StringVar(&r.logPrefix, "log-prefix", "", "the log line's prefix")
+	f.StringVar(&r.raw, "raw", "", "an nftables rule written by hand instead of the match flags")
+	f.StringVar(&r.comment, "comment", "", "a comment")
+	f.BoolVar(&r.disabled, "disabled", false, "keep the rule but don't use it")
+	f.BoolVar(&r.everyPacket, "every-packet", false, "check every packet, so a drop or reject also stops connections already open")
+}
+
+var amountRe = regexp.MustCompile(`^(\d+)\s*([a-zA-Z]*)$`)
+
+// byteUnit reads a unit: "" (packets, for rates), bytes, kbytes, mbytes,
+// gbytes, or b, kb, mb, gb in any case.
+func byteUnit(u string) (string, bool) {
+	switch strings.ToLower(u) {
+	case "", "packets", "packet", "p":
+		return "", true
+	case "bytes", "byte", "b":
+		return "bytes", true
+	case "kbytes", "kb", "k":
+		return "kbytes", true
+	case "mbytes", "mb", "m":
+		return "mbytes", true
+	case "gbytes", "gb", "g":
+		return "gbytes", true
+	}
+	return "", false
+}
+
+func parseLimit(s string) (*model.Limit, error) {
+	if s == "" {
+		return nil, nil
+	}
+	parts := strings.Split(s, ",")
+	rate, per, ok := strings.Cut(strings.TrimSpace(parts[0]), "/")
+	m := amountRe.FindStringSubmatch(strings.TrimSpace(rate))
+	unit, unitOK := "", false
+	if m != nil {
+		unit, unitOK = byteUnit(m[2])
+	}
+	if !ok || m == nil || !unitOK || unit == "gbytes" {
+		return nil, fmt.Errorf("--limit %q: write it as 10/minute, or as bytes 1mbytes/second (bytes, kbytes, mbytes), with ,burst=N ,over ,per-source", s)
+	}
+	n, _ := strconv.Atoi(m[1])
+	l := &model.Limit{Rate: n, Unit: unit, Per: strings.TrimSuffix(per, "s")}
+	for _, p := range parts[1:] {
+		k, v, _ := strings.Cut(strings.TrimSpace(p), "=")
+		switch k {
+		case "burst":
+			b := amountRe.FindStringSubmatch(v)
+			bu, buOK := "", false
+			if b != nil {
+				bu, buOK = byteUnit(b[2])
+			}
+			if b == nil || !buOK || (b[2] != "" && bu != unit) {
+				return nil, fmt.Errorf("--limit: burst=%q: a number, in the rate's unit", v)
+			}
+			l.Burst, _ = strconv.Atoi(b[1])
+		case "over":
+			l.Over = true
+		case "per-source":
+			l.PerSource = true
+		case "per-destination":
+			l.PerDestination = true
+		default:
+			return nil, fmt.Errorf("--limit: unknown %q (burst=N, over, per-source, per-destination)", k)
+		}
+	}
+	return l, nil
+}
+
+// parseQuota reads --quota: 5000mbytes,over,per-source,monthly=15.
+func parseQuota(s string) (*model.Quota, error) {
+	if s == "" {
+		return nil, nil
+	}
+	parts := strings.Split(s, ",")
+	m := amountRe.FindStringSubmatch(strings.TrimSpace(parts[0]))
+	unit, ok := "", false
+	if m != nil {
+		unit, ok = byteUnit(m[2])
+	}
+	if m == nil || !ok || unit == "" {
+		return nil, fmt.Errorf("--quota %q: write it as 5000mbytes or 5gbytes, with ,over ,per-source ,daily ,weekly or ,monthly=DAY", s)
+	}
+	n, _ := strconv.ParseInt(m[1], 10, 64)
+	q := &model.Quota{Bytes: n * model.ByteUnits[unit]}
+	for _, p := range parts[1:] {
+		k, v, _ := strings.Cut(strings.TrimSpace(p), "=")
+		switch k {
+		case "over":
+			q.Over = true
+		case "per-source":
+			q.PerSource = true
+		case "per-destination":
+			q.PerDestination = true
+		case "daily", "weekly":
+			q.Reset = k
+		case "monthly":
+			q.Reset = k
+			if v != "" {
+				d, err := strconv.Atoi(v)
+				if err != nil {
+					return nil, fmt.Errorf("--quota: monthly=%q: the day of the month, 1 to 31", v)
+				}
+				q.Day = d
+			}
+		default:
+			return nil, fmt.Errorf("--quota: unknown %q (over, per-source, per-destination, daily, weekly, monthly=DAY)", k)
+		}
+	}
+	return q, nil
+}
+
+// humanBytes writes bytes for people: "1.8 GB".
+func humanBytes(n int64) string {
+	switch {
+	case n >= 1<<30:
+		return strconv.FormatFloat(float64(n)/(1<<30), 'f', 1, 64) + " GB"
+	case n >= 1<<20:
+		return strconv.FormatFloat(float64(n)/(1<<20), 'f', 1, 64) + " MB"
+	case n >= 1<<10:
+		return strconv.FormatFloat(float64(n)/(1<<10), 'f', 1, 64) + " KB"
+	}
+	return strconv.FormatInt(n, 10) + " bytes"
+}
+
+func (r *ruleFlags) rule(chain string) (model.Rule, error) {
+	rule := model.Rule{Chain: chain, Match: r.match(), Action: r.action, RejectWith: r.rejectWith, CtState: r.states, CtStatus: r.statuses,
+		Days: r.days, ConnLimit: r.connLimit, EveryPacket: r.everyPacket, Log: r.log, LogPrefix: r.logPrefix, Raw: r.raw, Comment: r.comment, Disabled: r.disabled}
+	if r.hours != "" {
+		from, to, ok := strings.Cut(r.hours, "-")
+		if !ok {
+			return rule, fmt.Errorf("--hours: write it as 08:00-17:30")
+		}
+		rule.HourFrom, rule.HourTo = from, to
+	}
+	l, err := parseLimit(r.limit)
+	if err != nil {
+		return rule, err
+	}
+	rule.Limit = l
+	rule.Quota, err = parseQuota(r.quota)
+	rule.GroupV4, rule.GroupV6 = r.groupV4, r.groupV6
+	return rule, err
+}
+
+func ruleCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "rule", Aliases: []string{"rules"}, Short: "Rules: what to accept, drop or reject, in order"}
+	ls := &cobra.Command{
+		Use: "list [CHAIN]", Short: "List the rules, in order, with their hits", Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			query := q()
+			if len(args) == 1 {
+				query = q("chain", args[0])
+			}
+			rs, err := get[[]model.Rule]("/rules", query)
+			if err != nil {
+				return err
+			}
+			cs := counters()
+			return output(rs, func(w io.Writer) {
+				fmt.Fprintln(w, "CHAIN\tID\tMATCH\tACTION\tHITS\tCOMMENT")
+				for _, r := range rs {
+					what := describe(r.Match, r.Chain)
+					if r.Raw != "" {
+						what = "raw: " + r.Raw
+					}
+					extra := ruleExtras(r)
+					action := r.Action
+					if r.Raw != "" {
+						action = "-" // the text's own
+					}
+					if r.Log {
+						action += "+log"
+					}
+					if r.Disabled {
+						action += " (disabled)"
+					}
+					fmt.Fprintf(w, "%s\t%d\t%s%s\t%s\t%s\t%s\n", r.Chain, r.ID, what, extra, action, hits(cs, fmt.Sprintf("r%d", r.ID)), r.Comment)
+				}
+			})
+		},
+	}
+	cmd.AddCommand(ls)
+	cmd.AddCommand(showCmd("rules", "rule", func(w io.Writer, r model.Rule) {
+		if r.Raw == "" {
+			fmt.Fprintf(w, "ID:\t%d\nChain:\t%s\nMatch:\t%s%s\nAction:\t%s\n", r.ID, r.Chain, describe(r.Match, r.Chain), ruleExtras(r), r.Action)
+		} else {
+			fmt.Fprintf(w, "ID:\t%d\nChain:\t%s\n", r.ID, r.Chain)
+		}
+		if r.Raw != "" {
+			fmt.Fprintf(w, "Raw:\t%s\n", r.Raw)
+		}
+		if r.Log {
+			fmt.Fprintf(w, "Log:\tyes %s\n", r.LogPrefix)
+		}
+		if r.Comment != "" {
+			fmt.Fprintf(w, "Comment:\t%s\n", r.Comment)
+		}
+		cs := counters()
+		fmt.Fprintf(w, "Hits:\t%s\n", hits(cs, fmt.Sprintf("r%d", r.ID)))
+	}))
+	var rf ruleFlags
+	add := &cobra.Command{
+		Use: "add CHAIN [flags]", Short: "Add a rule (at the end of its chain unless --first, --before or --after)", Args: cobra.ExactArgs(1),
+		Long: `Adds a rule to a chain: input (to the firewall), forward (through it),
+output (from it) or early (before connection tracking: cheap drops).
+
+  hc rule add input --from trusted --service ssh
+  hc rule add input --in wg --service dns
+  hc rule add forward --in lan --out wan --from 2001:db8:1::/64
+  hc rule add input --in wan --service ssh --limit 3/minute,burst=5,over,per-source -a drop --log
+  hc rule add forward --in wan -a drop --comment "nothing else gets in"
+  hc rule add forward --raw 'iifname != "wan" ct state invalid ip daddr @trusted_v4 accept'`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			r, err := rf.rule(args[0])
+			if err != nil {
+				return err
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			// Placed in the same change: one history entry, and never in
+			// force at the end of the chain.
+			q := url.Values{}
+			switch {
+			case rf.first:
+				q.Set("first", "1")
+			case rf.before != 0:
+				q.Set("before", fmt.Sprint(rf.before))
+			case rf.after != 0:
+				q.Set("after", fmt.Sprint(rf.after))
+			}
+			var resp api.MutationResponse
+			if err := c.Do(ctx(), "POST", "/rules", q, r, &resp); err != nil {
+				return err
+			}
+			return report(&resp)
+		},
+	}
+	rf.register(add.Flags())
+	add.Flags().BoolVar(&rf.first, "first", false, "put it first in its chain")
+	add.Flags().Int64Var(&rf.before, "before", 0, "put it before rule ID")
+	add.Flags().Int64Var(&rf.after, "after", 0, "put it after rule ID")
+	cmd.AddCommand(add)
+	var uf ruleFlags
+	update := &cobra.Command{
+		Use: "update ID [flags]", Short: "Change a rule (only the flags given)", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fl := cmd.Flags()
+			p := patch{}
+			uf.matchFlags.patch(fl, p)
+			r, err := uf.rule("")
+			if err != nil {
+				return err
+			}
+			for flag, field := range map[string][2]any{
+				"action": {"action", r.Action}, "reject-with": {"reject_with", r.RejectWith}, "state": {"ct_state", r.CtState},
+				"status": {"ct_status", r.CtStatus}, "days": {"days", r.Days}, "conn-limit": {"conn_limit", r.ConnLimit},
+				"log": {"log", r.Log}, "log-prefix": {"log_prefix", r.LogPrefix}, "raw": {"raw", r.Raw}, "comment": {"comment", r.Comment},
+				"disabled": {"disabled", r.Disabled}, "group-v4": {"group_v4", r.GroupV4}, "group-v6": {"group_v6", r.GroupV6},
+				"every-packet": {"every_packet", r.EveryPacket},
+			} {
+				if fl.Changed(flag) {
+					p[field[0].(string)] = field[1]
+				}
+			}
+			if fl.Changed("hours") {
+				p["hour_from"], p["hour_to"] = r.HourFrom, r.HourTo
+			}
+			if fl.Changed("limit") {
+				p["limit"] = r.Limit
+			}
+			if fl.Changed("quota") {
+				p["quota"] = r.Quota
+			}
+			return change("PATCH", objPath("rules", args[0]), p)
+		},
+	}
+	uf.register(update.Flags())
+	cmd.AddCommand(update)
+	var before, after int64
+	var first, last bool
+	var chain string
+	move := &cobra.Command{
+		Use: "move ID (--before ID | --after ID | --first | --last | --chain CHAIN)", Short: "Move a rule", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			mv := api.MoveInput{Before: before, After: after, Chain: chain}
+			switch {
+			case first:
+				mv.To = "start"
+			case last:
+				mv.To = "end"
+			}
+			return change("POST", objPath("rules", args[0])+"/move", mv)
+		},
+	}
+	move.Flags().Int64Var(&before, "before", 0, "before rule ID")
+	move.Flags().Int64Var(&after, "after", 0, "after rule ID")
+	move.Flags().BoolVar(&first, "first", false, "first in its chain")
+	move.Flags().BoolVar(&last, "last", false, "last in its chain")
+	move.Flags().StringVar(&chain, "chain", "", "to the end of another chain")
+	cmd.AddCommand(move)
+	var resetAddr, give string
+	resetCmd := &cobra.Command{
+		Use: "reset-quota ID [--address ADDR] [--give AMOUNT]", Short: "Start a rule's quota again now, or give some back until its next reset", Args: cobra.ExactArgs(1),
+		Long: `Starts a rule's quota again now: all of it (every address's, for a
+per-address quota), or with --address one address's. --give takes an
+amount off what was used instead: extra allowance until the quota's next
+reset, without changing the rule. Done at once, not staged or on
+probation.
+
+  hc rule reset-quota 31
+  hc rule reset-quota 31 --address 192.168.1.23
+  hc rule reset-quota 31 --give 5gbytes --address 192.168.1.23`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := api.QuotaReset{Address: resetAddr}
+			if give != "" {
+				bad := fmt.Errorf("--give %q: write it as 500mbytes or 5gbytes", give)
+				m := amountRe.FindStringSubmatch(strings.TrimSpace(give))
+				if m == nil {
+					return bad
+				}
+				unit, ok := byteUnit(m[2])
+				n, _ := strconv.ParseInt(m[1], 10, 64)
+				if !ok || unit == "" || n < 1 {
+					return bad
+				}
+				in.Give = n * model.ByteUnits[unit]
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			if err := c.Do(ctx(), "POST", objPath("rules", args[0])+"/reset-quota", nil, in, nil); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				who := ""
+				if resetAddr != "" {
+					who = " for " + resetAddr
+				}
+				if in.Give > 0 {
+					fmt.Printf("gave back %s of the quota of rule %s%s, until its next reset\n", humanBytes(in.Give), args[0], who)
+				} else {
+					fmt.Printf("reset the quota of rule %s%s\n", args[0], who)
+				}
+			}
+			return nil
+		},
+	}
+	resetCmd.Flags().StringVar(&resetAddr, "address", "", "only this address's (a per-address quota; an IPv6 address counts for its /64)")
+	resetCmd.Flags().StringVar(&give, "give", "", "take this much off what was used instead (5gbytes, 500mbytes)")
+	cmd.AddCommand(resetCmd)
+	for _, on := range []bool{true, false} {
+		verb, short := "disable", "Keep a rule but don't use it"
+		if on {
+			verb, short = "enable", "Use a disabled rule again"
+		}
+		cmd.AddCommand(&cobra.Command{Use: verb + " ID", Short: short, Args: cobra.ExactArgs(1),
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return change("PATCH", objPath("rules", args[0]), patch{"disabled": !on})
+			}})
+	}
+	cmd.AddCommand(deleteCmd("rules", "rule"))
+	return cmd
+}
+
+func ruleExtras(r model.Rule) string {
+	var x []string
+	if len(r.CtState) > 0 {
+		x = append(x, "state "+strings.Join(r.CtState, ","))
+	}
+	if len(r.CtStatus) > 0 {
+		x = append(x, "status "+strings.Join(r.CtStatus, ","))
+	}
+	if len(r.Days) > 0 {
+		x = append(x, strings.Join(r.Days, ","))
+	}
+	if r.HourFrom != "" {
+		x = append(x, r.HourFrom+"-"+r.HourTo)
+	}
+	if l := r.Limit; l != nil {
+		unit := ""
+		if l.Unit != "" && l.Unit != "packets" {
+			unit = " " + l.Unit
+		}
+		s := fmt.Sprintf("limit %d%s/%s", l.Rate, unit, l.Per)
+		if l.Over {
+			s = "over " + s
+		}
+		if l.PerSource {
+			s += " per source"
+		}
+		if l.PerDestination {
+			s += " per destination"
+		}
+		x = append(x, s)
+	}
+	if q := r.Quota; q != nil {
+		s := "quota " + humanBytes(q.Bytes)
+		if q.Over {
+			s = "over " + s
+		}
+		if q.PerSource {
+			s += " per source"
+		}
+		if q.PerDestination {
+			s += " per destination"
+		}
+		switch {
+		case q.Reset == "monthly" && q.Day > 1:
+			s += fmt.Sprintf(" monthly (%d)", q.Day)
+		case q.Reset != "":
+			s += " " + q.Reset
+		}
+		x = append(x, s)
+	}
+	if r.ConnLimit > 0 {
+		x = append(x, fmt.Sprintf("over %d connections", r.ConnLimit))
+	}
+	if r.EveryPacket {
+		x = append(x, "every packet")
+	}
+	if len(x) == 0 {
+		return ""
+	}
+	return "  " + strings.Join(x, "  ")
+}
+
+// ---- port forwards
+
+func forwardCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "forward", Aliases: []string{"forwards", "port-forward"}, Short: "Port forwards: publish an inside host's service"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the port forwards, with their hits", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fs, err := get[[]model.PortForward]("/forwards", nil)
+			if err != nil {
+				return err
+			}
+			cs := counters()
+			return output(fs, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tNAME\tPUBLIC\tTO\tTO6\tFROM\tHAIRPIN\tHITS")
+				for _, f := range fs {
+					pub := list(f.PublicAddresses)
+					if len(f.PublicAddresses) == 0 {
+						pub = "(external zones)"
+					}
+					to := orDash(f.To)
+					if f.ToPort != "" {
+						to += ":" + f.ToPort
+					}
+					name := orDash(f.Name)
+					if f.Disabled {
+						name += " (disabled)"
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s %s/%s\t%s\t%s\t%s\t%s\t%s\n", f.ID, name, pub, strings.Join(f.Protocols, ","), f.Ports, to,
+						orDash(f.To6), list(f.Sources), yesNo(f.Hairpin), hits(cs, fmt.Sprintf("f%d", f.ID), fmt.Sprintf("f%d-6", f.ID)))
+				}
+			})
+		},
+	})
+	cmd.AddCommand(showCmd("forwards", "forward", func(w io.Writer, f model.PortForward) {
+		fmt.Fprintf(w, "ID:\t%d\nName:\t%s\nProtocols:\t%s\nPorts:\t%s\nPublic:\t%s\nTo:\t%s\n", f.ID, orDash(f.Name), list(f.Protocols), f.Ports, list(f.PublicAddresses), orDash(f.To))
+		if f.ToPort != "" {
+			fmt.Fprintf(w, "To port:\t%s\n", f.ToPort)
+		}
+		hairpin := yesNo(f.Hairpin)
+		if f.Hairpin && len(f.HairpinSources) > 0 {
+			hairpin += ", from " + list(f.HairpinSources)
+		}
+		fmt.Fprintf(w, "To (IPv6):\t%s\nFrom:\t%s\nHairpin:\t%s\n", orDash(f.To6), list(f.Sources), hairpin)
+		if f.Comment != "" {
+			fmt.Fprintf(w, "Comment:\t%s\n", f.Comment)
+		}
+		cs := counters()
+		fmt.Fprintf(w, "Hits:\t%s\n", hits(cs, fmt.Sprintf("f%d", f.ID), fmt.Sprintf("f%d-6", f.ID)))
+	}))
+	var pf model.PortForward
+	var to string
+	add := &cobra.Command{
+		Use: "add PORTS --to HOST[:PORT] [flags]", Short: "Add a port forward", Args: cobra.ExactArgs(1),
+		Long: `Forwards a public port (or range) to an inside host, lets the connections
+through, and with --hairpin makes the public address work from inside too.
+--to6 also opens the port to an IPv6 host (no translation there: its own
+port).
+
+  hc forward add 8443 --to 192.168.1.10:443 --to6 2001:db8:1::10 --public 203.0.113.10 --hairpin --name web
+  hc forward add 27015-27030 --proto tcp,udp --to 192.168.1.20 --public 203.0.113.10 --hairpin --name game
+  hc forward add 25 --to 192.168.1.25 --from trusted`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			pf.Ports = args[0]
+			if to != "" {
+				host, port, ok := strings.Cut(to, ":")
+				if ok && strings.Count(to, ":") == 1 {
+					pf.To, pf.ToPort = host, port
+				} else {
+					pf.To = to
+				}
+			}
+			return change("POST", "/forwards", pf)
+		},
+	}
+	f := add.Flags()
+	f.StringVar(&pf.Name, "name", "", "a name")
+	f.StringSliceVar(&pf.Protocols, "proto", []string{"tcp"}, "tcp, udp or both")
+	f.StringArrayVar(&pf.PublicAddresses, "public", nil, "a public IPv4 address or set to match (repeat; default: any address on the external zones)")
+	f.StringArrayVar(&pf.Sources, "from", nil, "only from these sources (repeat)")
+	f.StringVar(&to, "to", "", "the inside IPv4 host, with :PORT if it differs")
+	f.StringVar(&pf.To6, "to6", "", "an inside IPv6 host to open the port to")
+	f.BoolVar(&pf.Hairpin, "hairpin", false, "make the public address work from inside")
+	f.StringArrayVar(&pf.HairpinSources, "hairpin-from", nil, "only for these inside sources (repeat; default: all inside)")
+	f.StringVar(&pf.Comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	cmd.AddCommand(objectUpdate("forwards", "forward", map[string]string{"name": "name", "proto": "protocols", "public": "public_addresses",
+		"from": "sources", "to6": "to6", "hairpin": "hairpin", "hairpin-from": "hairpin_sources", "comment": "comment", "disabled": "disabled",
+		"ports": "ports", "to": "to", "to-port": "to_port"}, func(f *pflag.FlagSet) {
+		f.String("name", "", "a name")
+		f.StringSlice("proto", nil, "tcp, udp or both")
+		f.StringArray("public", nil, "public addresses (replace)")
+		f.StringArray("from", nil, "sources (replace)")
+		f.String("ports", "", "the public port or range")
+		f.String("to", "", "the inside IPv4 host")
+		f.String("to-port", "", "the inside port (\"\" keeps the public one)")
+		f.String("to6", "", "the inside IPv6 host (\"\": none)")
+		f.Bool("hairpin", false, "make the public address work from inside")
+		f.StringArray("hairpin-from", nil, "hairpin sources (replace)")
+		f.String("comment", "", "a comment")
+		f.Bool("disabled", false, "keep it but don't use it")
+	}))
+	cmd.AddCommand(deleteCmd("forwards", "forward"))
+	return cmd
+}
+
+// objectUpdate makes an update command whose flags map one to one onto
+// fields.
+func objectUpdate(kind, what string, fields map[string]string, register func(*pflag.FlagSet)) *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "update " + strings.ToUpper(what) + " [flags]", Short: "Change a " + what + " (only the flags given)", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			p := patch{}
+			var err error
+			cmd.Flags().Visit(func(fl *pflag.Flag) {
+				field, ok := fields[fl.Name]
+				if !ok {
+					return
+				}
+				switch fl.Value.Type() {
+				case "bool":
+					p[field] = fl.Value.String() == "true"
+				case "stringSlice":
+					v, _ := cmd.Flags().GetStringSlice(fl.Name)
+					p[field] = v
+				case "stringArray":
+					v, _ := cmd.Flags().GetStringArray(fl.Name)
+					p[field] = v
+				default:
+					p[field] = fl.Value.String()
+				}
+			})
+			if err != nil {
+				return err
+			}
+			return change("PATCH", objPath(kind, args[0]), p)
+		},
+	}
+	register(cmd.Flags())
+	return cmd
+}
+
+// ---- source NAT
+
+func snatCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "snat", Aliases: []string{"masquerade"}, Short: "Source NAT: masquerade or rewrite outgoing sources"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the source NAT rules, in order", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			xs, err := get[[]model.SourceNAT]("/snats", nil)
+			if err != nil {
+				return err
+			}
+			cs := counters()
+			return output(xs, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tMATCH\tTO\tHITS\tCOMMENT")
+				for _, x := range xs {
+					to := x.To
+					if to == "" {
+						to = "masquerade"
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\n", x.ID, describe(x.Match, ""), to, hits(cs, fmt.Sprintf("s%d", x.ID)), x.Comment)
+				}
+			})
+		},
+	})
+	var mf matchFlags
+	var to, comment string
+	add := &cobra.Command{
+		Use: "add [flags]", Short: "Add a source NAT rule", Args: cobra.NoArgs,
+		Long: `Masquerades (or with --snat-to, rewrites to an address) the source of
+traffic leaving the firewall. IPv4 only, unless --family ipv6 (NAT66).
+
+  hc snat add --out wan --from trusted
+  hc snat add --out wg --not-out --from wg_net
+  hc snat add --out wan --from fd00:1::/48 --family ipv6 --snat-to 2001:db8::1`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return change("POST", "/snats", model.SourceNAT{Match: mf.match(), To: to, Comment: comment})
+		},
+	}
+	mf.register(add.Flags(), true, true)
+	add.Flags().StringVar(&to, "snat-to", "", "rewrite to this address instead of masquerading")
+	add.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	cmd.AddCommand(natUpdate("snats", "snat", true))
+	cmd.AddCommand(deleteCmd("snats", "snat"))
+	return cmd
+}
+
+// natUpdate changes a source NAT rule or redirect.
+func natUpdate(kind, what string, out bool) *cobra.Command {
+	var mf matchFlags
+	var to, comment string
+	var disabled bool
+	cmd := &cobra.Command{
+		Use: "update ID [flags]", Short: "Change a " + what + " (only the flags given)", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fl := cmd.Flags()
+			p := patch{}
+			mf.patch(fl, p)
+			if fl.Changed("snat-to") {
+				p["to"] = to
+			}
+			if fl.Changed("to-port") {
+				p["to_port"] = to
+			}
+			if fl.Changed("comment") {
+				p["comment"] = comment
+			}
+			if fl.Changed("disabled") {
+				p["disabled"] = disabled
+			}
+			return change("PATCH", objPath(kind, args[0]), p)
+		},
+	}
+	mf.register(cmd.Flags(), true, out)
+	if out {
+		cmd.Flags().StringVar(&to, "snat-to", "", "rewrite to this address (\"\": masquerade)")
+	} else {
+		cmd.Flags().StringVar(&to, "to-port", "", "the local port")
+	}
+	cmd.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.Flags().BoolVar(&disabled, "disabled", false, "keep it but don't use it")
+	return cmd
+}
+
+// ---- redirects
+
+func redirectCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "redirect", Aliases: []string{"redirects"}, Short: "Redirects: send traffic to a port on the firewall itself"}
+	cmd.AddCommand(&cobra.Command{
+		Use: "list", Short: "List the redirects", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			xs, err := get[[]model.Redirect]("/redirects", nil)
+			if err != nil {
+				return err
+			}
+			cs := counters()
+			return output(xs, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tMATCH\tTO PORT\tHITS\tCOMMENT")
+				for _, x := range xs {
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\n", x.ID, describe(x.Match, ""), orDash(x.ToPort), hits(cs, fmt.Sprintf("d%d", x.ID)), x.Comment)
+				}
+			})
+		},
+	})
+	var mf matchFlags
+	var toPort, comment string
+	add := &cobra.Command{
+		Use: "add [flags]", Short: "Add a redirect", Args: cobra.NoArgs,
+		Long: `Sends matching traffic to a port on the firewall itself, so every client
+uses its DNS server, say:
+
+  hc redirect add --in lan --service dns --not-to 192.168.1.5 --comment "force DNS"`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return change("POST", "/redirects", model.Redirect{Match: mf.match(), ToPort: toPort, Comment: comment})
+		},
+	}
+	mf.register(add.Flags(), true, false)
+	add.Flags().StringVar(&toPort, "to-port", "", "the local port (default: the service's)")
+	add.Flags().StringVar(&comment, "comment", "", "a comment")
+	cmd.AddCommand(add)
+	cmd.AddCommand(natUpdate("redirects", "redirect", false))
+	cmd.AddCommand(deleteCmd("redirects", "redirect"))
+	return cmd
+}

cmd/hc/stats.go

@@ -0,0 +1,417 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+// Statistics: the figures housecarld keeps over time (docs/stats.md). The
+// types mirror internal/stats's JSON, so hc doesn't link the database.
+
+type statEntry struct {
+	Key   string `json:"key"`
+	Count int64  `json:"count"`
+	Name  string `json:"name,omitempty"`
+}
+
+type statTraffic struct {
+	Name     string `json:"name"`
+	Received int64  `json:"received"`
+	Sent     int64  `json:"sent"`
+}
+
+type statSummary struct {
+	Since           string        `json:"since"`
+	From            time.Time     `json:"from"`
+	Zones           []statTraffic `json:"zones"`
+	Interfaces      []statTraffic `json:"interfaces"`
+	Drops           int64         `json:"drops"`
+	DropsByChain    []statEntry   `json:"drops_by_chain"`
+	Rules           []statEntry   `json:"rules"`
+	PeakConnections int64         `json:"peak_connections"`
+	ConntrackMax    int64         `json:"conntrack_max"`
+	BlockedSources  []statEntry   `json:"blocked_sources"`
+	BlockedPorts    []statEntry   `json:"blocked_ports"`
+	DevicesUp       []statEntry   `json:"devices_up"`
+	DevicesDown     []statEntry   `json:"devices_down"`
+}
+
+type statTimeline struct {
+	Step   string  `json:"step"`
+	Times  []int64 `json:"times"`
+	Series []struct {
+		Key    string  `json:"key"`
+		Values []int64 `json:"values"`
+	} `json:"series"`
+}
+
+type statStatus struct {
+	Enabled     bool       `json:"enabled"`
+	Database    string     `json:"database"`
+	Size        int64      `json:"size"`
+	LastSample  *time.Time `json:"last_sample"`
+	SampleError string     `json:"sample_error"`
+	FirstData   *time.Time `json:"first_data"`
+	PerDevice   bool       `json:"per_device"`
+	Retention   struct {
+		Minute, Hour, Day string
+	} `json:"retention"`
+	Latest *struct {
+		Conntrack    int64   `json:"conntrack"`
+		ConntrackMax int64   `json:"conntrack_max"`
+		RSS          int64   `json:"rss"`
+		CPUPermille  int64   `json:"cpu_permille"`
+		MemUsed      int64   `json:"mem_used"`
+		Load         float64 `json:"load"`
+	} `json:"latest"`
+}
+
+func statsCmd() *cobra.Command {
+	var since string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "stats", Short: "Traffic, drops, blocked sources and devices over time",
+		Long: `Sums up what the firewall did over a while (--since: 90m, 24h, 7d, 4w):
+traffic per zone, packets dropped, the busiest rules, the most connections
+at once, the sources blocked most and what they tried, and (with traffic
+per device on) the devices that used the most.
+
+  hc stats --since 7d
+  hc stats traffic --since 24h
+  hc stats top devices --since 30d
+  hc stats rules --unused`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s, err := get[statSummary]("/stats/summary", q("since", since, "limit", strconv.Itoa(limit)))
+			if err != nil {
+				return err
+			}
+			return output(s, func(w io.Writer) { printSummary(w, s) })
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d, 4w…")
+	cmd.Flags().IntVar(&limit, "limit", 5, "entries per top list")
+	cmd.AddCommand(statsStatusCmd(), statsTrafficCmd(), statsTopCmd(), statsRulesCmd(), statsConnectionsCmd())
+	return cmd
+}
+
+func printSummary(w io.Writer, s statSummary) {
+	fmt.Fprintf(w, "Since %s (%s)\n\n", s.From.Local().Format("2006-01-02 15:04"), s.Since)
+	if len(s.Zones) > 0 {
+		fmt.Fprintln(w, "ZONE\tRECEIVED\tSENT")
+		for _, t := range s.Zones {
+			fmt.Fprintf(w, "%s\t%s\t%s\n", t.Name, humanBytes(t.Received), humanBytes(t.Sent))
+		}
+		fmt.Fprintln(w)
+	}
+	var chains []string
+	for _, e := range s.DropsByChain {
+		chains = append(chains, fmt.Sprintf("%s %s", e.Key, humanCount(uint64(e.Count))))
+	}
+	drops := humanCount(uint64(s.Drops))
+	if len(chains) > 0 {
+		drops += " (" + strings.Join(chains, ", ") + ")"
+	}
+	fmt.Fprintf(w, "Dropped:\t%s packets\n", drops)
+	if s.ConntrackMax > 0 {
+		fmt.Fprintf(w, "Connections:\tat most %d at once (of %d)\n", s.PeakConnections, s.ConntrackMax)
+	}
+	list := func(title string, es []statEntry, unit func(int64) string) {
+		if len(es) == 0 {
+			return
+		}
+		fmt.Fprintf(w, "\n%s\n", title)
+		for _, e := range es {
+			name := e.Key
+			if e.Name != "" {
+				name = e.Name
+			}
+			fmt.Fprintf(w, "  %s\t%s\n", name, unit(e.Count))
+		}
+	}
+	packets := func(n int64) string { return humanCount(uint64(n)) + " packets" }
+	list("Busiest rules:", s.Rules, packets)
+	list("Blocked most (logged drops):", s.BlockedSources, packets)
+	list("What they tried:", s.BlockedPorts, packets)
+	list("Devices, sent:", s.DevicesUp, humanBytes)
+	list("Devices, received:", s.DevicesDown, humanBytes)
+}
+
+func statsStatusCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "status", Short: "Whether statistics are collected, since when, and the database", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			s, err := get[statStatus]("/stats/status", nil)
+			if err != nil {
+				return err
+			}
+			return output(s, func(w io.Writer) {
+				if !s.Enabled {
+					fmt.Fprintln(w, "Statistics are off (stats.enabled in housecarl.yaml).")
+					return
+				}
+				fmt.Fprintf(w, "Collecting:\tyes\n")
+				if s.Database != "" {
+					fmt.Fprintf(w, "Database:\t%s (%s)\n", s.Database, humanBytes(s.Size))
+				} else {
+					fmt.Fprintf(w, "Database size:\t%s\n", humanBytes(s.Size))
+				}
+				if s.FirstData != nil {
+					fmt.Fprintf(w, "Since:\t%s\n", s.FirstData.Local().Format("2006-01-02 15:04"))
+				}
+				if s.LastSample != nil {
+					fmt.Fprintf(w, "Last sample:\t%s\n", s.LastSample.Local().Format("15:04:05"))
+				}
+				if s.SampleError != "" {
+					fmt.Fprintf(w, "Error:\t%s\n", s.SampleError)
+				}
+				fmt.Fprintf(w, "Kept:\tper minute %s, hourly %s, daily %s\n", s.Retention.Minute, s.Retention.Hour, s.Retention.Day)
+				fmt.Fprintf(w, "Per device:\t%s\n", map[bool]string{true: "on", false: "off (hc settings set --stats-per-device on)"}[s.PerDevice])
+				if l := s.Latest; l != nil {
+					if l.ConntrackMax > 0 {
+						fmt.Fprintf(w, "Connections now:\t%d of %d\n", l.Conntrack, l.ConntrackMax)
+					}
+					fmt.Fprintf(w, "housecarld:\t%s, CPU %.1f%%\n", humanBytes(l.RSS), float64(l.CPUPermille)/10)
+					fmt.Fprintf(w, "Host:\t%s in use, load %.2f\n", humanBytes(l.MemUsed), l.Load)
+				}
+			})
+		},
+	}
+}
+
+func statsTrafficCmd() *cobra.Command {
+	var since, step string
+	var ifaces bool
+	cmd := &cobra.Command{
+		Use: "traffic", Short: "Bytes received from and sent to each zone (or interface) over time", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			rx, tx := "zrx", "ztx"
+			if ifaces {
+				rx, tx = "rx", "tx"
+			}
+			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", rx+":*,"+tx+":*"))
+			if err != nil {
+				return err
+			}
+			return output(tl, func(w io.Writer) { printTimeline(w, tl, rx, tx) })
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
+	cmd.Flags().BoolVar(&ifaces, "interfaces", false, "per interface instead of per zone")
+	return cmd
+}
+
+// printTimeline writes a row per bucket: each name's received and sent.
+func printTimeline(w io.Writer, tl statTimeline, rx, tx string) {
+	var names []string
+	vals := map[string][]int64{}
+	for _, s := range tl.Series {
+		vals[s.Key] = s.Values
+		if n, ok := strings.CutPrefix(s.Key, rx+":"); ok {
+			names = append(names, n)
+		}
+	}
+	for _, s := range tl.Series {
+		if n, ok := strings.CutPrefix(s.Key, tx+":"); ok && vals[rx+":"+n] == nil {
+			names = append(names, n)
+		}
+	}
+	if len(names) == 0 {
+		fmt.Fprintln(w, "no traffic recorded yet")
+		return
+	}
+	fmt.Fprint(w, "TIME")
+	for _, n := range names {
+		fmt.Fprintf(w, "\t%s ↓\t%s ↑", n, n)
+	}
+	fmt.Fprintln(w)
+	layout := "01-02 15:04"
+	if tl.Step == "1d" {
+		layout = "2006-01-02"
+	}
+	at := func(k string, i int) int64 {
+		if v := vals[k]; i < len(v) {
+			return v[i]
+		}
+		return 0
+	}
+	for i, t := range tl.Times {
+		fmt.Fprint(w, time.Unix(t, 0).Local().Format(layout))
+		for _, n := range names {
+			fmt.Fprintf(w, "\t%s\t%s", humanBytes(at(rx+":"+n, i)), humanBytes(at(tx+":"+n, i)))
+		}
+		fmt.Fprintln(w)
+	}
+}
+
+func statsTopCmd() *cobra.Command {
+	var since string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "top devices|blocked|tried|quota RULE-ID", Short: "The largest: devices by traffic, sources blocked, what they tried, a quota's users",
+		Long: `Top lists over the last while:
+
+  devices   inside addresses by traffic to and from external zones (needs
+            hc settings set --stats-per-device on)
+  blocked   sources of logged drops (the policies' drops with 'log drops'
+            on, and rules that log)
+  tried     what those drops tried: tcp/22, udp/53, icmp/echo-request
+  quota ID  who used a per-address quota of rule ID`,
+		Args: cobra.RangeArgs(1, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			get1 := func(kind string) ([]statEntry, error) {
+				return get[[]statEntry]("/stats/top", q("kind", kind, "since", since, "limit", strconv.Itoa(limit)))
+			}
+			switch args[0] {
+			case "devices":
+				up, err := get1("device_up")
+				if err != nil {
+					return err
+				}
+				down, err := get1("device_down")
+				if err != nil {
+					return err
+				}
+				out := map[string][2]int64{}
+				var order []string
+				for _, l := range [][]statEntry{down, up} {
+					for _, e := range l {
+						if _, ok := out[e.Key]; !ok {
+							out[e.Key] = [2]int64{}
+							order = append(order, e.Key)
+						}
+					}
+				}
+				for _, e := range down {
+					v := out[e.Key]
+					v[0] = e.Count
+					out[e.Key] = v
+				}
+				for _, e := range up {
+					v := out[e.Key]
+					v[1] = e.Count
+					out[e.Key] = v
+				}
+				names := lookupNames(order...)
+				return output(map[string]any{"up": up, "down": down}, func(w io.Writer) {
+					fmt.Fprintln(w, "DEVICE\tRECEIVED\tSENT")
+					for _, k := range order {
+						fmt.Fprintf(w, "%s\t%s\t%s\n", named(names, k, k), humanBytes(out[k][0]), humanBytes(out[k][1]))
+					}
+				})
+			case "blocked", "tried":
+				kind := map[string]string{"blocked": "blocked_src", "tried": "blocked_port"}[args[0]]
+				es, err := get1(kind)
+				if err != nil {
+					return err
+				}
+				var names map[string]string
+				if args[0] == "blocked" {
+					names = lookupNames(keys(es)...)
+				}
+				return output(es, func(w io.Writer) {
+					fmt.Fprintln(w, map[string]string{"blocked": "SOURCE", "tried": "WHAT"}[args[0]]+"\tPACKETS")
+					for _, e := range es {
+						fmt.Fprintf(w, "%s\t%d\n", named(names, e.Key, e.Key), e.Count)
+					}
+				})
+			case "quota":
+				if len(args) != 2 {
+					return fmt.Errorf("give the rule's ID: hc stats top quota 12")
+				}
+				es, err := get1("quota_r" + args[1])
+				if err != nil {
+					return err
+				}
+				names := lookupNames(keys(es)...)
+				return output(es, func(w io.Writer) {
+					fmt.Fprintln(w, "ADDRESS\tUSED")
+					for _, e := range es {
+						fmt.Fprintf(w, "%s\t%s\n", named(names, e.Key, e.Key), humanBytes(e.Count))
+					}
+				})
+			}
+			return fmt.Errorf("unknown list %q: devices, blocked, tried or quota", args[0])
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().IntVar(&limit, "limit", 20, "at most this many")
+	return cmd
+}
+
+// keys are a top list's keys.
+func keys(es []statEntry) []string {
+	var out []string
+	for _, e := range es {
+		out = append(out, e.Key)
+	}
+	return out
+}
+
+func statsRulesCmd() *cobra.Command {
+	var since string
+	var unused bool
+	cmd := &cobra.Command{
+		Use: "rules", Short: "Packets each rule, built-in and policy counted; --unused: rules that matched nothing", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			es, err := get[[]statEntry]("/stats/hits", q("since", since))
+			if err != nil {
+				return err
+			}
+			if unused {
+				var keep []statEntry
+				for _, e := range es {
+					if e.Count == 0 && strings.HasPrefix(e.Key, "r") {
+						keep = append(keep, e)
+					}
+				}
+				es = keep
+			}
+			return output(es, func(w io.Writer) {
+				if unused && len(es) == 0 {
+					fmt.Fprintf(w, "every rule matched something in the last %s\n", since)
+					return
+				}
+				fmt.Fprintln(w, "WHAT\tPACKETS")
+				for _, e := range es {
+					fmt.Fprintf(w, "%s\t%d\n", orDash(e.Name), e.Count)
+				}
+			})
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "30d", "the last 24h, 7d, 30d…")
+	cmd.Flags().BoolVar(&unused, "unused", false, "only the rules that matched nothing")
+	return cmd
+}
+
+func statsConnectionsCmd() *cobra.Command {
+	var since, step string
+	cmd := &cobra.Command{
+		Use: "connections", Short: "How many connections the firewall tracked over time, against the limit", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			tl, err := get[statTimeline]("/stats/timeline", q("since", since, "step", step, "keys", "g:conntrack,g:conntrack_max"))
+			if err != nil {
+				return err
+			}
+			return output(tl, func(w io.Writer) {
+				fmt.Fprintln(w, "TIME\tCONNECTIONS\tLIMIT")
+				for i, t := range tl.Times {
+					n, m := tl.Series[0].Values[i], tl.Series[1].Values[i]
+					if n < 0 {
+						continue // no sample
+					}
+					fmt.Fprintf(w, "%s\t%d\t%d\n", time.Unix(t, 0).Local().Format("01-02 15:04"), n, m)
+				}
+			})
+		},
+	}
+	cmd.Flags().StringVar(&since, "since", "24h", "the last 90m, 24h, 7d…")
+	cmd.Flags().StringVar(&step, "step", "auto", "per 1m, 5m, 15m, 1h, 6h or 1d")
+	return cmd
+}

cmd/hc/traffic.go

@@ -0,0 +1,367 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/spf13/cobra"
+
+	"housecarl/internal/api"
+)
+
+// Traffic as it happens: connections the kernel tracks, and tracing
+// packets through the ruleset.
+
+func connectionsCmd() *cobra.Command {
+	var search string
+	var limit int
+	cmd := &cobra.Command{
+		Use: "connections", Aliases: []string{"conns"}, Short: "The connections the firewall tracks, and closing them",
+		Long: `Lists the connections the kernel tracks: what the "established" built-in
+lets through without checking the rules again. A rule added later doesn't
+stop them by itself; close them, and their next packet is checked by the
+rules as it stands.
+
+  hc connections --search 192.168.1.50
+  hc connections --search ":443 tcp"
+  hc connections close --search 192.168.1.50`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cs, err := get[[]api.Connection]("/connections", q("q", search, "limit", strconv.Itoa(limit)))
+			if err != nil {
+				return err
+			}
+			var addrs []string
+			for _, c := range cs {
+				addrs = append(addrs, c.Source, c.Destination)
+			}
+			names := lookupNames(addrs...)
+			return output(cs, func(w io.Writer) {
+				fmt.Fprintln(w, "PROTO\tFROM\tTO\tNAT\tSTATE\tEXPIRES")
+				for _, c := range cs {
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", c.Protocol, named(names, c.Source, hostPort(c.Source, c.SourcePort)),
+						named(names, c.Destination, hostPort(c.Destination, c.DestinationPort)), natText(c), c.State, fmtSeconds(c.Timeout))
+				}
+			})
+		},
+	}
+	cmd.Flags().StringVar(&search, "search", "", "only those with all these words: addresses, :port, protocol, state")
+	cmd.Flags().IntVar(&limit, "limit", 500, "at most this many")
+	var csearch string
+	var yes bool
+	closeCmd := &cobra.Command{
+		Use: "close --search TEXT", Short: "Close connections: the rules check them again with their next packet", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if strings.TrimSpace(csearch) == "" {
+				return fmt.Errorf("say which: --search 192.168.1.50 (see 'hc connections --search' first)")
+			}
+			if !confirm("Close every connection matching "+strconvQuote(csearch)+"?", yes) {
+				return fmt.Errorf("nothing closed")
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var out map[string]int
+			if err := c.Do(ctx(), "POST", "/connections/close", nil, api.CloseConnections{Search: csearch}, &out); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Printf("closed %s\n", plural(out["closed"], "connection"))
+			}
+			return nil
+		},
+	}
+	closeCmd.Flags().StringVar(&csearch, "search", "", "the connections to close, as for 'hc connections --search'")
+	closeCmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	cmd.AddCommand(closeCmd)
+	return cmd
+}
+
+func strconvQuote(s string) string { return strconv.Quote(s) }
+
+func hostPort(addr string, port int) string {
+	if port == 0 {
+		return addr
+	}
+	if strings.Contains(addr, ":") {
+		return fmt.Sprintf("[%s]:%d", addr, port)
+	}
+	return fmt.Sprintf("%s:%d", addr, port)
+}
+
+// natText: "to 192.168.1.10:443" (a port forward), "as 203.0.113.10"
+// (masqueraded).
+func natText(c api.Connection) string {
+	var parts []string
+	if c.DNATTo != "" {
+		port := c.DNATPort
+		if port == 0 {
+			port = c.DestinationPort
+		}
+		parts = append(parts, "to "+hostPort(c.DNATTo, port))
+	}
+	if c.SNATTo != "" {
+		parts = append(parts, "as "+c.SNATTo)
+	}
+	return strings.Join(parts, ", ")
+}
+
+func traceCmd() *cobra.Command {
+	var in api.TraceInput
+	cmd := &cobra.Command{
+		Use: "trace", Short: "Follow packets through every table's rules: what each met, and what decided",
+		Long: `Traces packets from or to an address, on a port (either way), through the
+whole ruleset, Housecarl's and every other table's, for a while: each
+rule and policy they meet, and what accepted or dropped them. Ctrl-C
+stops it.
+
+  hc trace --address 192.168.1.50 --port 443
+  hc trace --port 22 --seconds 120
+  hc trace --address 2001:db8:1::/64 --protocol icmpv6`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var st api.TraceStatus
+			if err := c.Do(ctx(), "POST", "/trace", nil, in, &st); err != nil {
+				return err
+			}
+			if !flagQuiet {
+				fmt.Fprintf(os.Stderr, "tracing until %s (Ctrl-C stops)\n", st.Ends.Local().Format("15:04:05"))
+			}
+			sig, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+			defer stop()
+			shown := 0
+			for {
+				select {
+				case <-sig.Done():
+					c.Do(ctx(), "DELETE", "/trace", nil, nil, &st)
+				case <-time.After(time.Second):
+					if err := c.Do(ctx(), "GET", "/trace", nil, nil, &st); err != nil {
+						return err
+					}
+				}
+				for ; shown < len(st.Packets); shown++ {
+					printPacket(os.Stdout, st.Packets[shown])
+				}
+				if !st.Running || sig.Err() != nil {
+					break
+				}
+			}
+			if st.Error != "" {
+				return fmt.Errorf("the trace failed: %s", st.Error)
+			}
+			if !flagQuiet {
+				fmt.Fprintf(os.Stderr, "%d packets traced\n", len(st.Packets))
+			}
+			return nil
+		},
+	}
+	f := cmd.Flags()
+	f.StringVar(&in.Address, "address", "", "packets from or to this address or prefix")
+	f.IntVar(&in.Port, "port", 0, "packets from or to this port")
+	f.StringVar(&in.Protocol, "protocol", "", "tcp, udp, sctp, icmp or icmpv6 (default any)")
+	f.IntVar(&in.Seconds, "seconds", 60, "how long (5 to 600)")
+	return cmd
+}
+
+// printPacket writes a traced packet: its headers, each step, its fate.
+func printPacket(w io.Writer, p api.TracePacket) {
+	what := p.Protocol
+	if p.TCPFlags != "" {
+		what += " " + p.TCPFlags
+	}
+	if p.ICMPType != "" {
+		what += " " + p.ICMPType
+	}
+	where := ""
+	if p.In != "" {
+		where += " in " + p.In
+	}
+	if p.Out != "" {
+		where += " out " + p.Out
+	}
+	fmt.Fprintf(w, "%s  %s -> %s  %s%s\n", p.Time.Local().Format("15:04:05.000"), hostPort(p.Source, p.SourcePort), hostPort(p.Destination, p.DestinationPort), what, where)
+	for _, s := range p.Steps {
+		name := s.Label
+		if name == "" {
+			if s.Kind == "policy" {
+				name = s.Chain + " policy"
+			} else {
+				name = s.Rule
+			}
+		}
+		fmt.Fprintf(w, "    %s %s: %s -> %s\n", s.Table, s.Chain, name, s.Verdict)
+	}
+	decided := p.DecidedBy.Label
+	if decided == "" {
+		decided = strings.TrimSpace(p.DecidedBy.Table + " " + p.DecidedBy.Chain + " " + p.DecidedBy.Kind)
+	}
+	if decided == "" {
+		fmt.Fprintf(w, "    => %s\n", strings.ToUpper(p.Verdict))
+	} else {
+		fmt.Fprintf(w, "    => %s by %s\n", strings.ToUpper(p.Verdict), decided)
+	}
+	if wd := p.WouldDrop; wd != nil {
+		name := wd.Label
+		if name == "" {
+			name = strings.TrimSpace(wd.Chain + " " + wd.Mark)
+		}
+		fmt.Fprintf(w, "    => shadow mode: Housecarl would have dropped it, by %s\n", name)
+	}
+}
+
+func matrixCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "matrix [FROM TO]", Short: "What the rules do to new connections between zones",
+		Long: `Shows, for each pair of zones (and the firewall itself), what happens to a
+new connection: allowed or blocked, decided by a rule or the policy, and
+how many rules decide for part of the traffic first (+N: exceptions,
+such as some services or addresses). With FROM and TO (a zone, or
+"firewall"), that pair's exceptions in order, and what decides the rest.
+
+  hc matrix
+  hc matrix guest wan
+  hc matrix lan firewall`,
+		Args: cobra.RangeArgs(0, 2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 1 {
+				return fmt.Errorf("give both: FROM and TO")
+			}
+			m, err := get[api.Matrix]("/matrix", nil)
+			if err != nil {
+				return err
+			}
+			if len(args) == 2 {
+				for _, c := range m.Cells {
+					if strings.EqualFold(c.From, args[0]) && strings.EqualFold(c.To, args[1]) {
+						return output(c, func(w io.Writer) { printCell(w, c) })
+					}
+				}
+				return fmt.Errorf("no pair %s → %s (zones: %s, and firewall)", args[0], args[1], strings.Join(m.Zones, ", "))
+			}
+			return output(m, func(w io.Writer) { printMatrix(w, m) })
+		},
+	}
+}
+
+func printMatrix(w io.Writer, m api.Matrix) {
+	writeMatrix(w, m, isTerminal(os.Stdout) && flagOutput == "table")
+}
+
+// writeMatrix writes the grid, in colour on a terminal.
+func writeMatrix(w io.Writer, m api.Matrix, color bool) {
+	cols := append(append([]string{}, m.Zones...), "firewall")
+	cells := map[[2]string]api.MatrixCell{}
+	for _, c := range m.Cells {
+		cells[[2]string{c.From, c.To}] = c
+	}
+	// The grid is padded by hand: tabwriter would count the colours'
+	// escape codes as text, and misalign the columns.
+	grid := [][]string{append([]string{"FROM \\ TO"}, cols...)}
+	kinds := [][]string{make([]string, len(cols)+1)}
+	same := false
+	for _, from := range cols {
+		row, kind := []string{from}, []string{""}
+		for _, to := range cols {
+			c, ok := cells[[2]string{from, to}]
+			if !ok {
+				row, kind = append(row, "-"), append(kind, "")
+				continue
+			}
+			text := map[string]string{"allowed": "allow", "blocked": "block", "partly": "partly"}[c.Summary]
+			if c.Allowed > 0 {
+				text += fmt.Sprintf(" +%d", c.Allowed)
+			}
+			if c.Blocked > 0 {
+				text += fmt.Sprintf(" -%d", c.Blocked)
+			}
+			if c.SameZone {
+				text += " *"
+				same = true
+			}
+			row, kind = append(row, text), append(kind, c.Summary)
+		}
+		grid, kinds = append(grid, row), append(kinds, kind)
+	}
+	width := make([]int, len(cols)+1)
+	for _, row := range grid {
+		for i, t := range row {
+			width[i] = max(width[i], utf8.RuneCountInString(t))
+		}
+	}
+	paint := map[string]string{"allowed": "\x1b[32m", "blocked": "\x1b[31m", "partly": "\x1b[33m"}
+	for r, row := range grid {
+		var b strings.Builder
+		for i, t := range row {
+			pad := strings.Repeat(" ", width[i]-utf8.RuneCountInString(t))
+			if i == len(row)-1 {
+				pad = ""
+			} else {
+				pad += "  "
+			}
+			if p := paint[kinds[r][i]]; color && p != "" {
+				t = p + t + "\x1b[0m"
+			}
+			b.WriteString(t + pad)
+		}
+		fmt.Fprintln(w, b.String())
+	}
+	// What the partly open pairs let through.
+	var open [][2]string
+	for _, from := range cols {
+		for _, to := range cols {
+			c, ok := cells[[2]string{from, to}]
+			if !ok || len(c.Open) == 0 || c.SameZone {
+				continue
+			}
+			items := c.Open
+			more := ""
+			if len(items) > 6 {
+				items, more = items[:6], fmt.Sprintf("; %d more", len(c.Open)-6)
+			}
+			open = append(open, [2]string{from + " → " + to, strings.Join(items, "; ") + more})
+		}
+	}
+	if len(open) > 0 {
+		fmt.Fprintln(w, "\nLet through for part of the traffic:")
+		for _, o := range open {
+			fmt.Fprintf(w, "  %s\t%s\n", o[0], o[1])
+		}
+	}
+	fmt.Fprintln(w, "\nallow, block: every new connection; partly: some are let through, some not.")
+	fmt.Fprintln(w, "+N: rules that let part of the traffic through; -N: rules that block part of it. 'hc matrix FROM TO' lists them.")
+	if same {
+		fmt.Fprintln(w, "*: a zone to itself: only traffic routed between its interfaces passes the firewall.")
+	}
+}
+
+func printCell(w io.Writer, c api.MatrixCell) {
+	summary := map[string]string{"allowed": "all allowed", "blocked": "all blocked", "partly": "partly allowed"}[c.Summary]
+	fmt.Fprintf(w, "%s → %s (the %s chain), new connections: %s\n", c.From, c.To, c.Chain, summary)
+	if c.Note != "" {
+		fmt.Fprintf(w, "  (%s)\n", c.Note)
+	}
+	hits := func(h *uint64) string {
+		if h == nil {
+			return "-"
+		}
+		return fmt.Sprintf("%d", *h)
+	}
+	fmt.Fprintln(w, "  VERDICT\tWHAT\tBY\tHITS")
+	for _, e := range c.Exceptions {
+		fmt.Fprintf(w, "  %s\t%s\t%s\t%s\n", e.Verdict, e.What, e.Label, hits(e.Hits))
+	}
+	fmt.Fprintf(w, "  %s\tthe rest\t%s\t%s\n", c.Verdict, c.DecidedBy, hits(c.Hits))
+	fmt.Fprintln(w, "\nHits: each rule's packets since housecarld started, in all its traffic (a rule can cover several pairs).")
+}

cmd/housecarld/lock.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// errInUse means another process holds the database lock.
+var errInUse = errors.New("the database is in use by a running housecarld or by 'housecarld restore'")
+
+// lockDatabase takes the exclusive lock (on DATABASE.lock) that marks the
+// database as in use by a running housecarld or by 'housecarld restore', so a
+// restore can never replace the database under a running housecarld, and two
+// housecarlds can't serve the same database. Other commands (backup, user, ...)
+// don't take it: they work alongside a running housecarld.
+func lockDatabase(db string) (release func(), err error) {
+	if err := os.MkdirAll(filepath.Dir(db), 0o750); err != nil {
+		return nil, err
+	}
+	path := db + ".lock"
+	// Read-only is enough for flock, and works whoever created the file.
+	f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0o640)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		f.Close()
+		if errors.Is(err, syscall.EWOULDBLOCK) {
+			return nil, errInUse
+		}
+		return nil, fmt.Errorf("locking %s: %w", path, err)
+	}
+	return func() {
+		syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
+}

cmd/housecarld/main.go

@@ -1,17 +1,54 @@
-// Command housecarld is Housecarl's daemon: it will keep the firewall's
-// nftables ruleset, and serve its REST API and web UI (docs/design.md).
-// For now it only knows its version.
+// Command housecarld is Housecarl's daemon: it keeps the firewall's
+// ruleset and serves the REST API and the web UI. It also has the commands
+// that work on the database directly (init, user, backup, restore, check).
 package main
 
 import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"log/slog"
+	"net"
 	"os"
+	"os/signal"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/term"
 
+	"housecarl/internal/auth"
+	"housecarl/internal/certs"
+	"housecarl/internal/config"
+	"housecarl/internal/conntrack"
+	"housecarl/internal/engine"
+	"housecarl/internal/hostif"
+	"housecarl/internal/jarl"
+	"housecarl/internal/jarlsets"
+	"housecarl/internal/linkwatch"
+	"housecarl/internal/model"
+	"housecarl/internal/names"
+	"housecarl/internal/nflog"
+	"housecarl/internal/nft"
+	"housecarl/internal/render"
+	"housecarl/internal/server"
+	"housecarl/internal/stats"
+	"housecarl/internal/store"
+	"housecarl/internal/tablewatch"
+	"housecarl/internal/trace"
+	"housecarl/internal/validate"
 	"housecarl/internal/version"
+	"housecarl/internal/web"
 )
 
+var configPath string
+
 func main() {
 	if err := rootCmd().Execute(); err != nil {
 		fmt.Fprintln(os.Stderr, "error:", err)
@@ -21,19 +58,744 @@ func main() {
 
 func rootCmd() *cobra.Command {
 	root := &cobra.Command{
-		Use:   "housecarld",
-		Short: "Housecarl's firewall daemon, API and web UI",
-		Long: `housecarld will keep a Linux firewall's nftables ruleset, built from its
-database, and serve the REST API and web UI that change it. It doesn't yet:
-Housecarl is at its start (docs/design.md).`,
+		Use:           "housecarld",
+		Short:         "Housecarl's firewall daemon, API and web UI",
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		Version:       version.String(),
 	}
-	root.AddCommand(versionCmd())
+	root.PersistentFlags().StringVarP(&configPath, "config", "c", config.DefaultPath, "configuration file")
+	root.AddCommand(serveCmd(), initCmd(), userCmd(), checkCmd(), backupCmd(), restoreCmd(), configCmd(), versionCmd())
 	return root
 }
 
+func newLogger(cfg *config.Config, w io.Writer) *slog.Logger {
+	var level slog.Level
+	switch cfg.Log.Level {
+	case "debug":
+		level = slog.LevelDebug
+	case "warn":
+		level = slog.LevelWarn
+	case "error":
+		level = slog.LevelError
+	}
+	opts := &slog.HandlerOptions{Level: level}
+	if cfg.Log.Format == "json" {
+		return slog.New(slog.NewJSONHandler(w, opts))
+	}
+	// The journal adds its own timestamps.
+	if os.Getenv("JOURNAL_STREAM") != "" {
+		opts.ReplaceAttr = func(groups []string, a slog.Attr) slog.Attr {
+			if a.Key == slog.TimeKey && len(groups) == 0 {
+				return slog.Attr{}
+			}
+			return a
+		}
+	}
+	return slog.New(slog.NewTextHandler(w, opts))
+}
+
+type app struct {
+	cfg   *config.Config
+	log   *slog.Logger
+	store *store.Store
+}
+
+func openApp(allowMissing bool) (*app, error) {
+	cfg, err := config.Load(configPath, allowMissing)
+	if err != nil {
+		return nil, fmt.Errorf("loading the configuration: %w", err)
+	}
+	log := newLogger(cfg, os.Stderr)
+	if err := os.MkdirAll(filepath.Dir(cfg.Database), 0o750); err != nil {
+		return nil, err
+	}
+	st, err := store.Open(cfg.Database)
+	if err != nil {
+		return nil, fmt.Errorf("opening the database %s: %w", cfg.Database, err)
+	}
+	return &app{cfg: cfg, log: log, store: st}, nil
+}
+
+// hostEnv describes this machine's interfaces for validation.
+func hostEnv(h *hostif.Host) func() *validate.Env {
+	return func() *validate.Env {
+		all, err := h.All()
+		if err != nil {
+			return nil
+		}
+		return &validate.Env{Interfaces: all}
+	}
+}
+
+func serveCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "serve",
+		Short: "Keep the firewall's ruleset, and serve the API and the web UI",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(true)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			release, err := lockDatabase(a.cfg.Database)
+			if err != nil {
+				return err
+			}
+			defer release()
+			ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+			defer stop()
+			a.log.Info("starting", "version", version.String(), "database", a.cfg.Database)
+			if users, err := a.store.ListUsers(ctx); err == nil && len(users) == 0 {
+				a.log.Warn("there are no users yet; create an administrator with 'housecarld init' (root can use 'hc' on the local socket meanwhile)")
+			}
+			eng, err := engine.New(ctx, a.store, a.log)
+			if err != nil {
+				return err
+			}
+			host := &hostif.Host{}
+			eng.Env = hostEnv(host)
+			fw := &nft.Firewall{Runner: &nft.Runner{Command: nft.Wrapper()}, Interfaces: host.Exists, BootFile: a.cfg.Ruleset,
+				Shadow: a.cfg.Shadow.Enabled, ShadowBefore: a.cfg.Shadow.Position == config.ShadowBefore,
+				Bans: func() []render.Ban { return server.ActiveBans(context.Background(), a.store) }}
+			eng.Firewall = fw
+			if fw.Shadow {
+				a.log.Warn("shadow mode: Housecarl's ruleset is loaded but blocks, rejects and translates nothing; the other firewall decides (docs/shadow.md)", "position", a.cfg.Shadow.Position)
+			}
+			for _, p := range eng.Problems() {
+				a.log.Warn("configuration problem: "+p.Message, "object", p.Object, "code", p.Code)
+			}
+			var jc *jarl.Client
+			if a.cfg.Jarl.Enabled() {
+				tok, err := a.cfg.Jarl.ReadToken()
+				if err != nil {
+					return err
+				}
+				if jc, err = jarl.New(a.cfg.Jarl.Server, tok, a.cfg.Jarl.CACert, a.cfg.Jarl.Insecure); err != nil {
+					return err
+				}
+			}
+			var acme *certs.Manager
+			if a.cfg.TLS.ACME.Domain != "" {
+				if jc == nil {
+					return errors.New("tls.acme needs Jarl: set jarl.server and its token (docs/certificates.md)")
+				}
+				acme = certs.New(ctx, certs.Config{ACME: a.cfg.TLS.ACME, Version: version.Short()}, jc, a.store, a.log)
+				go acme.Run(ctx)
+			}
+			// The sets whose addresses come from Jarl: what was last read
+			// is put in force at once, Jarl is read again soon after.
+			jarlSets := &jarlsets.Resolver{Interval: time.Duration(a.cfg.Jarl.Refresh), State: eng.State, Log: a.log,
+				Apply: func(ctx context.Context) error {
+					if err := eng.ApplyCurrent(ctx); err != nil {
+						return err
+					}
+					return fw.SaveBoot(eng.State())
+				},
+				Audit: func(ctx context.Context, set, detail string) {
+					eng.Audit(ctx, &auth.Principal{Method: "jarl"}, "set.jarl", set, "success", detail)
+				},
+				Load: func(ctx context.Context) (string, error) { return a.store.GetMeta(ctx, jarlsets.MetaKey) },
+				Save: func(ctx context.Context, v string) error { return a.store.SetMeta(ctx, jarlsets.MetaKey, v) }}
+			if jc != nil {
+				jarlSets.Source = jc
+			}
+			jarlSets.Restore(ctx)
+			fw.JarlSets = jarlSets
+			var resolver *names.Resolver
+			if a.cfg.Names.Enabled {
+				resolver = names.New(a.cfg.Names.Resolver, time.Duration(a.cfg.Names.Cache))
+			}
+			srv := server.New(server.Config{
+				Listen: a.cfg.Listen, TLSCertFile: a.cfg.TLS.CertFile, TLSKeyFile: a.cfg.TLS.KeyFile, TrustedProxies: a.cfg.Proxies(),
+				SessionLifetime: time.Duration(a.cfg.Session.Lifetime), SessionIdle: time.Duration(a.cfg.Session.IdleTimeout),
+				CookieSecure: a.cfg.Session.CookieSecure, Socket: a.cfg.Socket, ACME: acme,
+				Shadow: a.cfg.Shadow.Enabled, ShadowPosition: a.cfg.Shadow.Position, Names: resolver, JarlSets: jarlSets,
+			}, eng, fw, host, a.log, web.FS())
+			jarlSets.Connected = srv.Connected
+			// Put the ruleset in force: what was on probation when
+			// housecarld stopped is rolled back first. An empty
+			// configuration applies nothing: its default policies would
+			// close the machine before any rule lets anyone in.
+			// What the quotas used before housecarld (or the machine)
+			// stopped comes back with the next apply.
+			restoreQuotas := func() {
+				if v, err := a.store.GetMeta(ctx, server.QuotaUsageKey); err == nil && v != "" {
+					var u nft.QuotaUsage
+					if json.Unmarshal([]byte(v), &u) == nil {
+						fw.RestoreQuotas(&u)
+					}
+				}
+			}
+			restoreQuotas()
+			if !srv.ResumeProbation(ctx) {
+				if eng.State().Empty() {
+					a.log.Info("the configuration is empty: no ruleset is applied until the first change (or 'hc import')")
+				} else if err := eng.ApplyCurrent(ctx); err != nil {
+					a.log.Error("applying the ruleset failed; the one in force stays (see 'hc status')", "err", err)
+				} else if err := fw.SaveBoot(eng.State()); err != nil {
+					a.log.Error("saving the ruleset for the next boot failed", "err", err)
+				}
+			}
+			// A flowtable device that appears later (ppp0 once PPPoE is up,
+			// or again after it reconnects) is added by applying again.
+			go func() {
+				err := linkwatch.Watch(ctx, func() {
+					host.Forget()
+					jarlSets.Kick() // a zone's networks may have changed
+					if !fw.FlowtableStale(eng.State()) {
+						return
+					}
+					if err := eng.ApplyCurrent(ctx); err != nil {
+						a.log.Error("applying the ruleset again for the flowtable's devices failed", "err", err)
+						return
+					}
+					a.log.Info("interfaces changed: applied the ruleset again, with the flowtable's devices that exist now")
+				})
+				if err != nil {
+					a.log.Warn("not watching the interfaces: a flowtable device that appears later is only added by the next change", "err", err)
+				}
+			}()
+			// In shadow mode the other firewall's ruleset is reloaded now
+			// and then, and a ruleset file usually starts with flush
+			// ruleset, which deletes Housecarl's table too: put it back.
+			if fw.Shadow {
+				go func() {
+					err := tablewatch.Watch(ctx, func() {
+						if _, err := fw.Table(ctx); !errors.Is(err, nft.ErrNoTable) || eng.State().Empty() {
+							return
+						}
+						// The quotas' usage went with the table: from
+						// what was last saved.
+						restoreQuotas()
+						if err := eng.ApplyCurrent(ctx); err != nil {
+							a.log.Error("Housecarl's table was deleted, and putting it back failed", "err", err)
+							return
+						}
+						a.log.Info("Housecarl's table was deleted (flush ruleset?): put it back")
+					})
+					if err != nil {
+						a.log.Warn("not watching nftables: if Housecarl's table is deleted, only the next change or a restart puts it back", "err", err)
+					}
+				}()
+			}
+			go jarlSets.Run(ctx)
+			go nflog.Listen(ctx, render.LogGroup, a.log, srv.LogPacket)
+			go srv.RunQuotas(ctx)
+			srv.SetConntrack(conntrack.Client{})
+			statsDone := startStats(ctx, a, fw, srv)
+			// A trace left behind by a crash.
+			fw.DeleteTable(ctx, "inet", trace.Table)
+			hup := make(chan os.Signal, 1)
+			signal.Notify(hup, syscall.SIGHUP)
+			go func() {
+				for range hup {
+					sdNotify("RELOADING=1")
+					srv.ReloadTLS()
+					sdNotify("READY=1")
+				}
+			}()
+			err = srv.Serve(ctx, func() { sdNotify("READY=1") })
+			sdNotify("STOPPING=1")
+			srv.SaveQuotaUsage(context.Background())
+			<-statsDone
+			a.log.Info("stopped")
+			return err
+		},
+	}
+}
+
+// startStats opens the statistics database and runs the collector until
+// ctx ends; the channel closes once it wrote its last figures.
+func startStats(ctx context.Context, a *app, fw *nft.Firewall, srv *server.Server) <-chan struct{} {
+	done := make(chan struct{})
+	c := a.cfg.Stats
+	if !c.Enabled {
+		close(done)
+		return done
+	}
+	st, err := stats.OpenStore(c.Database)
+	if err != nil {
+		a.log.Error("statistics are off: opening their database failed (deleting it is safe)", "err", err)
+		close(done)
+		return done
+	}
+	opts := stats.Options{TopN: c.TopN, RetainMinute: time.Duration(c.Retention.Minute), RetainHour: time.Duration(c.Retention.Hour), RetainDay: time.Duration(c.Retention.Day)}
+	// Tests sample more often than every minute.
+	if v := os.Getenv("HOUSECARL_STATS_INTERVAL"); v != "" {
+		opts.Interval, _ = time.ParseDuration(v)
+	}
+	src := stats.Sources{Table: fw.Table}
+	srv.StatsSources(&src)
+	col := stats.NewCollector(st, opts, src, a.log)
+	srv.SetStats(col)
+	go func() {
+		defer close(done)
+		defer st.Close()
+		col.Run(ctx)
+	}()
+	return done
+}
+
+func sdNotify(state string) {
+	sock := os.Getenv("NOTIFY_SOCKET")
+	if sock == "" {
+		return
+	}
+	addr := &net.UnixAddr{Name: sock, Net: "unixgram"}
+	if strings.HasPrefix(sock, "@") {
+		addr.Name = "\x00" + sock[1:]
+	}
+	c, err := net.DialUnix("unixgram", nil, addr)
+	if err != nil {
+		return
+	}
+	defer c.Close()
+	c.Write([]byte(state))
+}
+
+// ---- init and users
+
+func readPassword(prompt string, fromStdin bool) (string, error) {
+	if fromStdin || !term.IsTerminal(int(os.Stdin.Fd())) {
+		line, err := bufio.NewReader(os.Stdin).ReadString('\n')
+		if err != nil && !errors.Is(err, io.EOF) {
+			return "", err
+		}
+		return strings.TrimRight(line, "\r\n"), nil
+	}
+	fmt.Fprint(os.Stderr, prompt)
+	b, err := term.ReadPassword(int(os.Stdin.Fd()))
+	fmt.Fprintln(os.Stderr)
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprint(os.Stderr, "Repeat: ")
+	b2, err := term.ReadPassword(int(os.Stdin.Fd()))
+	fmt.Fprintln(os.Stderr)
+	if err != nil {
+		return "", err
+	}
+	if string(b) != string(b2) {
+		return "", errors.New("the passwords don't match")
+	}
+	return string(b), nil
+}
+
+// readLine reads one line a byte at a time, so nothing after it is
+// buffered away from the password prompt that follows.
+func readLine(r io.Reader) string {
+	var b []byte
+	one := make([]byte, 1)
+	for {
+		n, err := r.Read(one)
+		if n == 1 {
+			if one[0] == '\n' {
+				break
+			}
+			b = append(b, one[0])
+		}
+		if err != nil {
+			break
+		}
+	}
+	return strings.TrimRight(string(b), "\r")
+}
+
+func localPrincipal(u *model.User) *auth.Principal {
+	return &auth.Principal{User: u, Method: "local", IP: "local"}
+}
+
+func audit(ctx context.Context, st *store.Store, p *auth.Principal, action, object string) {
+	ev := &model.AuditEvent{Action: action, Object: object, Result: "success", Auth: p.Method, Username: p.Name()}
+	if p.User != nil {
+		ev.UserID = p.User.ID
+	}
+	st.AddAudit(ctx, ev)
+}
+
+func createToken(ctx context.Context, st *store.Store, u *model.User, name string) (string, error) {
+	secret, pid, hash := auth.NewToken()
+	if err := st.CreateToken(ctx, &model.Token{UserID: u.ID, Name: name, PublicID: pid, SecretHash: hash}); err != nil {
+		return "", err
+	}
+	return secret, nil
+}
+
+func initCmd() *cobra.Command {
+	var username string
+	var stdin, withToken bool
+	cmd := &cobra.Command{
+		Use:   "init",
+		Short: "Create the database and the first administrator",
+		Long: `Creates Housecarl's database with a first administrator.
+
+Housecarl starts with an empty configuration, and applies no ruleset until
+it has one: import your nftables ruleset with 'hc import', or write rules
+with 'hc rule add' or in the web UI (docs/install.md).`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(true)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			ctx := context.Background()
+			users, err := a.store.ListUsers(ctx)
+			if err != nil {
+				return err
+			}
+			if len(users) > 0 {
+				return fmt.Errorf("Housecarl is already set up (%d users); add users with 'housecarld user add' or 'hc user add'", len(users))
+			}
+			// Asked for when not given, on a terminal.
+			if !cmd.Flags().Changed("user") && !stdin && term.IsTerminal(int(os.Stdin.Fd())) {
+				fmt.Fprint(os.Stderr, "The first administrator's user name [admin]: ")
+				if name := strings.TrimSpace(readLine(os.Stdin)); name != "" {
+					username = name
+				}
+			}
+			if err := server.ValidUsername(username); err != nil {
+				return err
+			}
+			pw, err := readPassword("Password for "+username+": ", stdin)
+			if err != nil {
+				return err
+			}
+			hash, err := auth.HashPassword(pw)
+			if err != nil {
+				return err
+			}
+			u := &model.User{Username: username, PasswordHash: hash, Admin: true, Comment: "created by housecarld init"}
+			if err := a.store.CreateUser(ctx, u); err != nil {
+				return err
+			}
+			audit(ctx, a.store, localPrincipal(u), "system.init", "user "+u.Username)
+			fmt.Printf("Created the administrator %q in %s.\n", u.Username, a.cfg.Database)
+			if withToken {
+				tok, err := createToken(ctx, a.store, u, "cli")
+				if err != nil {
+					return err
+				}
+				fmt.Printf("\nAPI token for the CLI (shown only once):\n\n  %s\n", tok)
+			}
+			fmt.Println("\nNext steps:")
+			fmt.Println("  1. Start the daemon:          sudo systemctl enable --now housecarl")
+			fmt.Println("  2. Import your ruleset:       sudo hc import --defines /etc/nftables.conf -n   (look first; docs/import.md)")
+			fmt.Println("     or let yourself in first:  sudo hc rule add input --from 10.0.10.0/24 --service ssh   (then zones, rules, NAT)")
+			fmt.Printf("  3. Open the web UI:           http://%s/ (see listen in %s)\n", a.cfg.Listen[0], configPath)
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&username, "user", "u", "admin", "the administrator's name (best your own: each person should have their own account); asked for on a terminal")
+	cmd.Flags().BoolVar(&stdin, "password-stdin", false, "read the password from standard input")
+	cmd.Flags().BoolVar(&withToken, "token", false, "also make an API token for the CLI")
+	return cmd
+}
+
+func userCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "user", Short: "Manage users directly in the database (when the API can't be used)"}
+	var admin, stdin bool
+	add := &cobra.Command{
+		Use: "add NAME", Short: "Add a user", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(true)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			if err := server.ValidUsername(args[0]); err != nil {
+				return err
+			}
+			pw, err := readPassword("Password for "+args[0]+": ", stdin)
+			if err != nil {
+				return err
+			}
+			hash, err := auth.HashPassword(pw)
+			if err != nil {
+				return err
+			}
+			u := &model.User{Username: args[0], PasswordHash: hash, Admin: admin}
+			if err := a.store.CreateUser(context.Background(), u); err != nil {
+				if errors.Is(err, store.ErrConflict) {
+					return fmt.Errorf("user %s already exists", args[0])
+				}
+				return err
+			}
+			audit(context.Background(), a.store, localPrincipal(&model.User{Username: "root"}), "user.create", "user "+u.Username)
+			fmt.Printf("added user %s\n", u.Username)
+			return nil
+		},
+	}
+	add.Flags().BoolVar(&admin, "admin", false, "an administrator")
+	add.Flags().BoolVar(&stdin, "password-stdin", false, "read the password from standard input")
+	cmd.AddCommand(add)
+	pw := &cobra.Command{
+		Use: "password NAME", Short: "Set a user's password (and log them out)", Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(true)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			ctx := context.Background()
+			u, err := a.store.GetUserByName(ctx, args[0])
+			if err != nil {
+				return fmt.Errorf("no user %s", args[0])
+			}
+			p, err := readPassword("New password for "+u.Username+": ", stdin)
+			if err != nil {
+				return err
+			}
+			if u.PasswordHash, err = auth.HashPassword(p); err != nil {
+				return err
+			}
+			u.Disabled = false
+			if err := a.store.UpdateUser(ctx, u); err != nil {
+				return err
+			}
+			a.store.DeleteUserSessions(ctx, u.ID, "")
+			audit(ctx, a.store, localPrincipal(&model.User{Username: "root"}), "user.password", "user "+u.Username)
+			fmt.Printf("set the password of %s\n", u.Username)
+			return nil
+		},
+	}
+	pw.Flags().BoolVar(&stdin, "password-stdin", false, "read the password from standard input")
+	cmd.AddCommand(pw)
+	cmd.AddCommand(&cobra.Command{
+		Use: "token NAME TOKEN-NAME", Short: "Make an API token for a user (printed once)", Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(true)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			ctx := context.Background()
+			u, err := a.store.GetUserByName(ctx, args[0])
+			if err != nil {
+				return fmt.Errorf("no user %s", args[0])
+			}
+			tok, err := createToken(ctx, a.store, u, args[1])
+			if err != nil {
+				return err
+			}
+			audit(ctx, a.store, localPrincipal(&model.User{Username: "root"}), "token.create", "token "+u.Username+"/"+args[1])
+			fmt.Println(tok)
+			return nil
+		},
+	})
+	return cmd
+}
+
+func checkCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "check",
+		Short: "Check the configuration in the database against this machine",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			a, err := openApp(false)
+			if err != nil {
+				return err
+			}
+			defer a.store.Close()
+			eng, err := engine.New(context.Background(), a.store, a.log)
+			if err != nil {
+				return err
+			}
+			eng.Env = hostEnv(&hostif.Host{})
+			ps := eng.Problems()
+			for _, p := range ps {
+				fmt.Printf("%s: %s: %s [%s]\n", p.Severity, p.Object, p.Message, p.Code)
+			}
+			if validate.HasErrors(ps) {
+				return errors.New("the configuration has errors")
+			}
+			if len(ps) == 0 {
+				fmt.Println("ok: no problems")
+			}
+			return nil
+		},
+	}
+}
+
+// ---- backups
+
+// backupName is a backup's file name; the time in it orders and prunes
+// backups.
+const backupPrefix, backupSuffix = "housecarl-", ".db"
+
+func backupCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "backup [FILE]",
+		Short: "Write a backup of the database (and delete old ones)",
+		Long: `Writes a consistent copy of the database, also while housecarld runs. Without
+FILE it goes to backup.dir as housecarl-<time>.db, and backups older than
+backup.retention are deleted (keeping at least backup.keep_at_least).`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return err
+			}
+			if _, err := os.Stat(cfg.Database); err != nil {
+				return fmt.Errorf("no database at %s yet (run 'housecarld init')", cfg.Database)
+			}
+			st, err := store.OpenExisting(cfg.Database)
+			if err != nil {
+				return err
+			}
+			defer st.Close()
+			dest := ""
+			if len(args) == 1 {
+				dest = args[0]
+			} else {
+				if err := os.MkdirAll(cfg.Backup.Dir, 0o700); err != nil {
+					return err
+				}
+				dest = filepath.Join(cfg.Backup.Dir, backupPrefix+time.Now().UTC().Format("20060102T150405Z")+backupSuffix)
+			}
+			tmp := dest + ".tmp"
+			os.Remove(tmp)
+			if err := st.Backup(context.Background(), tmp); err != nil {
+				return fmt.Errorf("writing the backup: %w", err)
+			}
+			if err := os.Rename(tmp, dest); err != nil {
+				return err
+			}
+			fmt.Println(dest)
+			if len(args) == 0 {
+				return prune(cfg)
+			}
+			return nil
+		},
+	}
+}
+
+func prune(cfg *config.Config) error {
+	entries, err := os.ReadDir(cfg.Backup.Dir)
+	if err != nil {
+		return err
+	}
+	type bk struct {
+		name string
+		t    time.Time
+	}
+	var list []bk
+	for _, e := range entries {
+		n := e.Name()
+		if !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, backupSuffix) {
+			continue
+		}
+		t, err := time.Parse("20060102T150405Z", strings.TrimSuffix(strings.TrimPrefix(n, backupPrefix), backupSuffix))
+		if err != nil {
+			continue
+		}
+		list = append(list, bk{n, t})
+	}
+	sort.Slice(list, func(i, j int) bool { return list[i].t.After(list[j].t) })
+	cut := time.Now().Add(-time.Duration(cfg.Backup.Retention))
+	for i, b := range list {
+		if i < cfg.Backup.KeepAtLeast || b.t.After(cut) {
+			continue
+		}
+		os.Remove(filepath.Join(cfg.Backup.Dir, b.name))
+	}
+	return nil
+}
+
+func restoreCmd() *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use:   "restore FILE",
+		Short: "Replace the database with a backup (housecarld must be stopped)",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return err
+			}
+			release, err := lockDatabase(cfg.Database)
+			if err != nil {
+				return fmt.Errorf("%w; stop it first: systemctl stop housecarl", err)
+			}
+			defer release()
+			src, err := store.OpenExisting(args[0])
+			if err != nil {
+				return fmt.Errorf("%s isn't a database: %w", args[0], err)
+			}
+			v, err := src.Version(context.Background())
+			if err != nil {
+				src.Close()
+				return fmt.Errorf("%s isn't a Housecarl database: %w", args[0], err)
+			}
+			var ok string
+			if err := src.DB().QueryRow(`PRAGMA integrity_check`).Scan(&ok); err != nil || ok != "ok" {
+				src.Close()
+				return fmt.Errorf("%s fails SQLite's integrity check: %s %v", args[0], ok, err)
+			}
+			src.Close()
+			if v > store.SchemaVersion() {
+				return fmt.Errorf("the backup is from a newer Housecarl (schema %d, this one has %d)", v, store.SchemaVersion())
+			}
+			if !yes {
+				fmt.Fprintf(os.Stderr, "Replace %s with %s? [y/N] ", cfg.Database, args[0])
+				var a string
+				fmt.Scanln(&a)
+				if a != "y" && a != "yes" {
+					return errors.New("not restored")
+				}
+			}
+			stamp := time.Now().UTC().Format("20060102T150405Z")
+			for _, ext := range []string{"", "-wal", "-shm"} {
+				if _, err := os.Stat(cfg.Database + ext); err == nil {
+					if err := os.Rename(cfg.Database+ext, cfg.Database+".before-restore-"+stamp+ext); err != nil {
+						return err
+					}
+				}
+			}
+			b, err := os.ReadFile(args[0])
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile(cfg.Database, b, 0o640); err != nil {
+				return err
+			}
+			st, err := store.Open(cfg.Database)
+			if err != nil {
+				return err
+			}
+			defer st.Close()
+			audit(context.Background(), st, localPrincipal(&model.User{Username: "root"}), "system.restore", args[0])
+			fmt.Printf("restored %s; the old database is %s.before-restore-%s\n", args[0], cfg.Database, stamp)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+	return cmd
+}
+
+func configCmd() *cobra.Command {
+	var example bool
+	cmd := &cobra.Command{
+		Use: "config", Short: "Show the configuration in use (--example: a commented example)", Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if example {
+				fmt.Print(config.Example)
+				return nil
+			}
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("listen:   %s\nsocket:   %s\ndatabase: %s\nbackups:  %s\n", strings.Join(cfg.Listen, ", "), cfg.Socket, cfg.Database, cfg.Backup.Dir)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVar(&example, "example", false, "print a commented example configuration")
+	return cmd
+}
+
 func versionCmd() *cobra.Command {
 	return &cobra.Command{
 		Use: "version", Short: "Print the version", Args: cobra.NoArgs,

contrib/fail2ban/action.d/housecarl.conf

@@ -0,0 +1,52 @@
+# Housecarl's action for fail2ban: a ban goes into a Housecarl ban list,
+# and the list says what the banned address is kept from (the firewall
+# itself, some hosts behind it, or everything). See docs/bans.md.
+#
+# Copy this file to /etc/fail2ban/action.d/housecarl.conf, make a ban
+# list and a token for it alone on the firewall:
+#
+#   hc banlist add web --scope hosts --host 10.0.10.57
+#   hc user add f2b-web --no-password --grant bans:write:web
+#   hc token create web --user f2b-web        # as root on the firewall
+#
+# put the token in /etc/fail2ban/housecarl.token (mode 0600), and use it
+# in a jail:
+#
+#   [sshd]
+#   enabled = true
+#   action  = housecarl[list=web, url="https://router.lan:8070"]
+#
+# The jail's port and protocol go along: port = ssh bans only tcp/22;
+# fail2ban's default (0:65535) bans every port. IPv6 addresses are banned
+# with the list's grouping (a /64 unless set otherwise). Housecarl never
+# bans inside networks, its own addresses or the list's "never".
+#
+# On the firewall itself, the local socket works too, still with the
+# token: url = http://localhost, curlopts = --unix-socket
+# /run/housecarl/housecarl.sock.
+
+[Definition]
+
+actionstart =
+actionstop =
+actioncheck =
+
+# A ban renews one already there; a bantime of -1 is the list's longest.
+actionban = <_curl> -f -X POST -H "Content-Type: application/json" "<url>/api/v1/banlists/<list>/bans"
+            -d '{"address": "<ip>", "protocol": "<protocol>", "ports": "<port>", "time": <bantime>, "source": "fail2ban/<name>"}'
+
+# A ban Housecarl already ended (its own time ran out) is no error.
+actionunban = <_curl> -X DELETE "<url>/api/v1/banlists/<list>/bans?address=<ip>"
+
+[Init]
+
+# The ban list, and Housecarl's URL (its listen address).
+list =
+url = https://127.0.0.1:8070
+# The API token, in a file only root reads.
+tokenfile = /etc/fail2ban/housecarl.token
+# More curl options: --cacert /etc/fail2ban/housecarl-ca.pem for a
+# private CA, --unix-socket for the local socket.
+curlopts =
+
+_curl = curl -sS --max-time 10 <curlopts> -o /dev/null -H "Authorization: Bearer $(cat <tokenfile>)"

deploy/debian/control-housecarl-cli.in

@@ -0,0 +1,17 @@
+Package: housecarl-cli
+Version: @VERSION@
+Architecture: @ARCH@
+Maintainer: @MAINTAINER@
+Installed-Size: @SIZE@
+Section: net
+Priority: optional
+Description: command line client for Housecarl, the firewall manager
+ hc is the command line client for Housecarl servers (housecarld): zones,
+ address sets, services, rules, port forwards, source NAT and redirects,
+ counters and the drop log, the history of changes with undo and the
+ confirmation of changes on probation, users and tokens, and importing an
+ nftables ruleset. It talks to housecarld's REST API with an API token,
+ or, on the firewall itself, to housecarld's local socket.
+ .
+ Install this package alone to manage Housecarl from another machine; the
+ housecarl package (the server) depends on it.

deploy/debian/control-housecarl.in

@@ -0,0 +1,22 @@
+Package: housecarl
+Version: @VERSION@
+Architecture: @ARCH@
+Maintainer: @MAINTAINER@
+Installed-Size: @SIZE@
+Depends: systemd, nftables, housecarl-cli (= @VERSION@)
+Section: net
+Priority: optional
+Description: nftables firewall manager with a web UI, a REST API and a CLI
+ Housecarl keeps a Linux firewall's nftables ruleset: zones, address sets,
+ services, ordered rules, port forwards with hairpin NAT and IPv6
+ pinholes, source NAT and redirects, IPv4 and IPv6 together, in its own
+ table (inet housecarl). It is configured through a web UI, a REST API and
+ the hc command line client. Every change is validated as a whole and
+ checked by nftables before it is applied in one transaction, recorded in
+ a history where it can be undone, and audited. A change made over the
+ network is rolled back unless confirmed. It can import an existing
+ nftables ruleset.
+ .
+ This package is the server: housecarld (as the systemd service housecarl),
+ the boot unit housecarl-ruleset and the daily backup timer. The hc CLI
+ comes from housecarl-cli.

deploy/debian/postinst

@@ -0,0 +1,63 @@
+#!/bin/sh
+# Housecarl: runs after the package's files are in place.
+set -e
+
+if [ "$1" = configure ]; then
+	# The housecarl system user and Housecarl's directories: /etc/housecarl,
+	# /var/lib/housecarl and /var/backups/housecarl.
+	systemd-sysusers housecarl.conf
+	systemd-tmpfiles --create housecarl.conf || true
+
+	# An example configuration on the first install; an existing one is
+	# never touched.
+	if [ ! -e /etc/housecarl/housecarl.yaml ]; then
+		/usr/bin/housecarld config --example > /etc/housecarl/housecarl.yaml.dpkg-new
+		chown root:housecarl /etc/housecarl/housecarl.yaml.dpkg-new
+		chmod 0640 /etc/housecarl/housecarl.yaml.dpkg-new
+		mv /etc/housecarl/housecarl.yaml.dpkg-new /etc/housecarl/housecarl.yaml
+	fi
+
+	# The daily database backup (housecarl-backup.timer) is on by default. An
+	# administrator's 'systemctl disable housecarl-backup.timer' is remembered
+	# by deb-systemd-helper and kept across upgrades. Before 'housecarld init'
+	# the backup skips itself (there is no database yet).
+	if command -v deb-systemd-helper >/dev/null 2>&1; then
+		deb-systemd-helper unmask housecarl-backup.timer >/dev/null || true
+		if deb-systemd-helper --quiet was-enabled housecarl-backup.timer; then
+			deb-systemd-helper enable housecarl-backup.timer >/dev/null || true
+		else
+			deb-systemd-helper update-state housecarl-backup.timer >/dev/null || true
+		fi
+	elif [ -z "$2" ] && [ -d /run/systemd/system ]; then
+		systemctl enable housecarl-backup.timer >/dev/null 2>&1 || true
+	fi
+
+	# The boot unit loads the last confirmed ruleset before the network comes
+	# up; it does nothing until housecarld has saved one.
+	if [ -z "$2" ] && [ -d /run/systemd/system ]; then
+		systemctl enable housecarl-ruleset.service >/dev/null 2>&1 || true
+	fi
+
+	if [ -d /run/systemd/system ]; then
+		systemctl daemon-reload || true
+		if systemctl is-enabled --quiet housecarl-backup.timer 2>/dev/null; then
+			systemctl start housecarl-backup.timer || true
+		fi
+		# An upgrade restarts a running housecarld (its database is upgraded
+		# when it starts). A first install starts nothing: Housecarl needs
+		# 'housecarld init' first.
+		if [ -n "$2" ] && systemctl is-active --quiet housecarl.service; then
+			systemctl restart housecarl.service || echo "housecarl: restarting housecarld failed; see 'journalctl -u housecarl'" >&2
+		fi
+	fi
+
+	if [ -z "$2" ]; then
+		echo "Housecarl is installed. Next: create the database and start it, as described"
+		echo "in /usr/share/doc/housecarl/docs/install.md:"
+		echo "  sudo -u housecarl housecarld init && sudo systemctl enable --now housecarl"
+		echo "To bring your current nftables ruleset over: sudo hc import --defines /etc/nftables.conf -n"
+		echo "(docs/import.md). Housecarl only writes its own table, inet housecarl."
+	fi
+fi
+
+exit 0

deploy/debian/postrm

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Housecarl: runs after the package's files are removed.
+set -e
+
+case "$1" in
+remove|purge)
+	if [ -d /run/systemd/system ]; then
+		systemctl daemon-reload || true
+	fi
+	;;
+esac
+
+# The backup timer's enabled state (see postinst): masked while the
+# package is removed, forgotten on purge.
+if command -v deb-systemd-helper >/dev/null 2>&1; then
+	case "$1" in
+	remove)
+		deb-systemd-helper mask housecarl-backup.timer >/dev/null || true
+		;;
+	purge)
+		deb-systemd-helper purge housecarl-backup.timer >/dev/null || true
+		deb-systemd-helper unmask housecarl-backup.timer >/dev/null || true
+		;;
+	esac
+fi
+
+# Even a purge keeps Housecarl's data: the database in /var/lib/housecarl holds
+# the configuration and its history. Remove it by hand when you no longer
+# need it.
+if [ "$1" = purge ]; then
+	echo "housecarl: kept /etc/housecarl, /var/lib/housecarl and /var/backups/housecarl;"
+	echo "remove them by hand if unused."
+fi
+
+exit 0

deploy/debian/prerm

@@ -0,0 +1,16 @@
+#!/bin/sh
+# Housecarl: runs before the package's files are removed (also on upgrades).
+set -e
+
+# On removal (not on upgrade) stop housecarld and the backup timer and remove
+# the boot-time links (postrm deals with the timer's). The ruleset in force
+# stays until the next boot; then the machine has no Housecarl table.
+if [ "$1" = remove ] && [ -d /run/systemd/system ]; then
+	systemctl stop housecarl.service housecarl-backup.timer housecarl-backup.service 2>/dev/null || true
+	systemctl disable housecarl.service housecarl-ruleset.service 2>/dev/null || true
+	if ! command -v deb-systemd-helper >/dev/null 2>&1; then
+		systemctl disable housecarl-backup.timer 2>/dev/null || true
+	fi
+fi
+
+exit 0

deploy/systemd/housecarl-backup.service

@@ -0,0 +1,44 @@
+# Housecarl: one database backup, run daily by housecarl-backup.timer.
+# Installed as /etc/systemd/system/housecarl-backup.service (see
+# docs/operations.md#backups). What to keep, and where, is in the backup:
+# section of /etc/housecarl/housecarl.yaml.
+[Unit]
+Description=Housecarl database backup
+Documentation=file:///usr/local/share/doc/housecarl/docs/operations.md
+# Nothing to back up before 'housecarld init'.
+ConditionPathExists=/var/lib/housecarl/housecarl.db
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/housecarld --config /etc/housecarl/housecarl.yaml backup
+User=housecarl
+Group=housecarl
+UMask=0077
+Nice=10
+IOSchedulingClass=idle
+
+# If you change backup.dir in housecarl.yaml, allow the new one too:
+# 'systemctl edit housecarl-backup.service' and add ReadWritePaths=/new/dir.
+ProtectSystem=strict
+ReadWritePaths=/var/lib/housecarl -/var/backups/housecarl
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+PrivateNetwork=yes
+NoNewPrivileges=yes
+CapabilityBoundingSet=
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_UNIX
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM

deploy/systemd/housecarl-backup.timer

@@ -0,0 +1,20 @@
+# Housecarl: runs housecarl-backup.service (a database backup) every day.
+# Installed as /etc/systemd/system/housecarl-backup.timer. Change the schedule
+# with 'systemctl edit housecarl-backup.timer', e.g.
+#   [Timer]
+#   OnCalendar=
+#   OnCalendar=*-*-* 02:30
+# and keep backup.retention in housecarl.yaml above the new interval.
+[Unit]
+Description=Daily Housecarl database backup
+Documentation=file:///usr/local/share/doc/housecarl/docs/operations.md
+
+[Timer]
+OnCalendar=daily
+# Spread the start over an hour, and catch up on a run missed while the
+# machine was off.
+RandomizedDelaySec=1h
+Persistent=true
+
+[Install]
+WantedBy=timers.target

deploy/systemd/housecarl-ruleset.service

@@ -0,0 +1,25 @@
+# Housecarl: loads the last confirmed ruleset at boot, before the network
+# comes up and whether or not housecarld starts. If it fails to load, the
+# fallback housecarld saved beside it is loaded instead: nothing new gets
+# in from external zones (docs/operations.md), and the unit is marked
+# failed, until housecarld starts and puts the real ruleset back.
+# Installed as /etc/systemd/system/housecarl-ruleset.service.
+[Unit]
+Description=Housecarl firewall ruleset
+Documentation=file:///usr/local/share/doc/housecarl/README.md
+DefaultDependencies=no
+Before=network-pre.target shutdown.target
+Wants=network-pre.target
+# Debian's nftables.service flushes the whole ruleset: Housecarl's loads after it.
+After=nftables.service local-fs.target
+Conflicts=shutdown.target
+ConditionPathExists=/var/lib/housecarl/ruleset.nft
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# "<2>": logged as critical.
+ExecStart=/bin/sh -c '/usr/sbin/nft -f /var/lib/housecarl/ruleset.nft && exit 0; echo "<2>the saved ruleset failed to load: loading the fallback (nothing new gets in from external zones) until housecarld puts the real one back" >&2; /usr/sbin/nft -f /var/lib/housecarl/ruleset-fallback.nft; exit 1'
+
+[Install]
+WantedBy=sysinit.target

deploy/systemd/housecarl.service

@@ -0,0 +1,63 @@
+# Housecarl: keeps the firewall's nftables ruleset; its REST API and web UI.
+# Installed as /etc/systemd/system/housecarl.service (see docs/install.md).
+[Unit]
+Description=Housecarl firewall manager (with web UI and API)
+Documentation=file:///usr/local/share/doc/housecarl/README.md
+# The confirmed ruleset is already in force from housecarl-ruleset.service.
+After=housecarl-ruleset.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=notify
+NotifyAccess=main
+ExecStart=/usr/local/bin/housecarld --config /etc/housecarl/housecarl.yaml serve
+# SIGHUP rereads the TLS certificate.
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=on-failure
+RestartSec=2s
+
+User=housecarl
+Group=housecarl
+UMask=0027
+
+# /var/lib/housecarl: the database and the confirmed ruleset (ruleset.nft).
+StateDirectory=housecarl
+StateDirectoryMode=0750
+ConfigurationDirectory=housecarl
+ConfigurationDirectoryMode=0750
+# /run/housecarl: the local API socket (housecarl.sock), for root and the
+# housecarl user.
+RuntimeDirectory=housecarl
+RuntimeDirectoryMode=0750
+
+# Sandboxing: housecarld writes only its own state.
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+NoNewPrivileges=yes
+# The only privilege housecarld gets: changing nftables (it runs nft) and
+# reading logged packets (nfnetlink_log), both CAP_NET_ADMIN.
+CapabilityBoundingSet=CAP_NET_ADMIN
+AmbientCapabilities=CAP_NET_ADMIN
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+# AF_NETLINK: nftables, logged packets, the interfaces and their addresses.
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+
+[Install]
+WantedBy=multi-user.target
+# 'systemctl restart housecarld' works too (the daemon's binary is housecarld).
+Alias=housecarld.service

deploy/sysusers.d/housecarl.conf

@@ -0,0 +1,2 @@
+# Housecarl's system user (installed as /usr/lib/sysusers.d/housecarl.conf).
+u housecarl - "Housecarl firewall manager" /var/lib/housecarl

deploy/tmpfiles.d/housecarl.conf

@@ -0,0 +1,8 @@
+# Directories Housecarl needs (installed as /usr/lib/tmpfiles.d/housecarl.conf).
+# Apply with: systemd-tmpfiles --create housecarl.conf
+#
+# Type Path                    Mode  User    Group
+d      /etc/housecarl            0750  root    housecarl
+d      /var/lib/housecarl        0750  housecarl housecarl
+# Database backups (housecarl-backup.timer; backup.dir in housecarl.yaml).
+d      /var/backups/housecarl    0700  housecarl housecarl

docs/api.md

@@ -0,0 +1,304 @@
+# The REST API
+
+Everything the web UI and the CLI do goes through this API, under
+`/api/v1`. Requests and answers are JSON.
+
+## Authentication
+
+* **API tokens**: `Authorization: Bearer housecarl_...`. Make one in the
+  web UI (your account), with `hc token create NAME`, or with `housecarld
+  user token USER NAME` on the server.
+* **Sessions**: `POST /auth/login` with `{"username", "password"}` sets a
+  cookie and returns `csrf_token`. Requests other than GET must send it as
+  `X-CSRF-Token`.
+* **The local socket** (`/run/housecarl/housecarl.sock`): root and
+  housecarld's own user need nothing; others send a token. Changes made
+  through it aren't on probation.
+
+Failed logins and token attempts are rate limited per address and per
+user name. [permissions.md](permissions.md) has what each call needs.
+
+## Errors
+
+```json
+{"error": {"code": "validation", "message": "rule input #31: there is no set trusted",
+           "problems": [{"severity": "error", "code": "unknown_set", "message": "from: there is no set trusted", "object": "rule input #31"}]}}
+```
+
+| Status | Code | |
+|---|---|---|
+| 400 | `bad_request` | the request can't be read |
+| 401 | `unauthenticated`, `session_expired`, `login_failed` | |
+| 403 | `forbidden`, `csrf` | |
+| 404 | `not_found` | |
+| 409 | `conflict` | e.g. undoing a change after the objects changed again; a takeover or closing connections in shadow mode ([shadow.md](shadow.md)) |
+| 422 | `validation` | the change would make the configuration invalid; `problems` lists why ([validation.md](validation.md)) |
+| 422 | `firewall` | nftables refused the ruleset (`nft -c`), or applying it failed; the message has nftables' words. Nothing was changed |
+| 429 | `rate_limited` | |
+
+## Changes
+
+Calls that change the configuration answer:
+
+```json
+{"applied": true, "history_id": 7,
+ "changes": [{"kind": "rule", "id": 31, "label": "input #31 (SSH from home)", "after": {...}}],
+ "warnings": [{"severity": "warning", "code": "set_empty", "message": "...", "object": "..."}],
+ "object": {...},
+ "confirm_by": "2026-10-19T10:02:00Z"}
+```
+
+* `confirm_by`: the change is on probation, and is rolled back at that
+  time unless `POST /confirm` comes first. Missing for changes made
+  through the socket, and in shadow mode ([shadow.md](shadow.md)).
+* `confirm_token`: the client that made the change can confirm it by
+  itself with `POST /confirm {"token"}`, from the same address, as the
+  same user and token, over a new connection (the answer closes this
+  one). Missing when a person must confirm (`manual_confirm`).
+* `?dry_run=1` checks the change, with nftables too, and returns what it
+  would do, with `"applied": false, "dry_run": true`, and `diff`: the
+  ruleset's unified diff, each `@@` line followed by the chain (or other
+  block of the table) the hunk starts in, as in `@@ -40,6 +40,8 @@ chain
+  forward`.
+* `?changeset=ID` (or the header `X-Housecarl-Changeset: ID`) stages the
+  change in that changeset instead ([Changesets](#changesets)): the answer
+  has `"staged": true`, `changes` (this operation's) and `changeset`.
+* `before` is missing for a create and `after` for a delete.
+* `warnings` are the problems the change brings; older ones are in
+  `/status`.
+* `object` is the created or changed object.
+
+Creating answers 201, other changes 200.
+
+## Endpoints
+
+### Status
+
+| | |
+|---|---|
+| `GET /healthz` | `ok`, `version`, `firewall` (`applied`, `unconfigured`, `fallback`, `missing`), `mode` (`enforcing`, or `shadow`: the ruleset blocks nothing, [shadow.md](shadow.md)), `probation` (changes waiting to be confirmed), `warnings` (codes); HTTP 503 unless `ok`. Without authentication; read at most every five seconds ([operations.md](operations.md#monitoring)) |
+| `GET /status` | `build`, `started`, `mode` (`enforcing` or `shadow`), `shadow_position` (`after` or `before`, in shadow mode), `applied` (the ruleset is in force), `apply_error`, `problems`, `probation`, `interfaces`, `counts` (objects by kind), `other_tables` (other nftables tables, `"inet main"`) |
+| `GET /interfaces` | the machine's interfaces: `name`, `up`, `addresses`, `zone` |
+| `GET /matrix[?changeset=ID]` | the zone matrix: `zones`, `cells` (`from`, `to` (a zone, or `firewall`), `chain`, `summary` (allowed, blocked, partly), `verdict` (what the rest gets: allowed, blocked), `decided_by`, `rule_id`, `hits`, `allowed` and `blocked` (exceptions by verdict), `open` (what the allowing exceptions let through, in short), `same_zone` and `note`, `exceptions`: `rule_id`, `chain`, `label`, `verdict` (allowed, blocked, logged, unknown), `what`, `short`, `hits`). `hits` are the rule's or policy's packets since housecarld started, all its traffic (missing: no counter) ([rules.md](rules.md#the-zone-matrix)) |
+| `GET /ruleset[?enforcing=1]` | the nftables ruleset Housecarl builds, as text. In shadow mode it is the shadow ruleset; `enforcing=1` gives the one loaded once shadow mode is off |
+| `GET /counters` | `{"since", "counters": {"r12": {"packets", "bytes"}, ...}}`, by mark ([design.md](design.md#its-own-table-only)); totals since housecarld started |
+| `GET /quotas` | each rule's quota: `rule_id`, `rule`, `quota`, `used` (bytes), `addresses` (`[{"address", "used"}]` for a per-address quota), `last_reset`, `next_reset` |
+| `GET /connections[?q=&limit=]` | tracked connections: `family`, `protocol`, `source`, `source_port`, `destination`, `destination_port` (or `icmp_id`, `icmp_type`, `icmp_code`), `dnat_to`, `dnat_port`, `snat_to`, `state`, `assured`, `timeout` (seconds left), `packets` and `bytes` when the kernel counts them. `q`: all of its words in the addresses, `:port`, protocol, state |
+| `POST /names` | `{"addresses": [...]}` (at most 2000): `{"enabled", "names": {"192.168.1.20": "laptop.example.net"}}`, the names of the inside addresses among them, by reverse lookup ([configuration.md](configuration.md#names)). Addresses without a name, outside ones, and those still being looked up after a second are left out; ask again for those. `log:read` |
+| `POST /connections/close` | `{"connections": [...]}` (as listed) or `{"search": "192.168.1.50"}`: delete their tracking entries (`config:write`, audited); answers `{"closed": N}`. 409 in shadow mode |
+| `POST /trace` | `{"address", "port", "protocol", "seconds"}` (5-600, default 60): start tracing (`config:write`, audited); 409 while one runs |
+| `GET /trace` | the trace running, or the last: `running`, `input`, `started`, `ends`, `error`, `packets`: each with `time`, `in`, `out`, `protocol`, addresses and ports, `tcp_flags`, `icmp_type`, `steps` (`table`, `chain`, `kind` rule or policy, `rule`, `mark`, `label`, `verdict`), `verdict` (accepted, dropped) and `decided_by`; in shadow mode `would_drop`, the step where Housecarl would have dropped it |
+| `DELETE /trace` | stop it |
+| `GET /log[?limit=&after=&q=]` | logged packets, newest first: `time`, `prefix`, `in`, `out`, `family`, `protocol`, `source`, `source_port`, `destination`, `destination_port`, `icmp_type`, `tcp_flags`, `length`, `mark` (the rule's), `verdict`, `shadow` (logged in shadow mode: a drop or reject is what would have been done). `after` (a time) for those since; `q` searches |
+| `GET /catalog` | built-in services, chains, actions, roles, the log group |
+
+### Settings
+
+`GET /settings`, `PATCH /settings` with any of:
+
+```json
+{"input_policy": "drop", "forward_policy": "drop", "output_policy": "accept",
+ "log_policy_drops": false,
+ "builtins": {"loopback": true, "established": true, "drop_invalid": true, "icmpv6": true,
+              "ping_internal": true, "dhcpv6_client": true, "antispoof": true, "mss_clamp": false},
+ "own_prefixes": ["2001:db8::/48"], "antispoof_v4": ["10.0.0.0/8", "..."], "antispoof_v6": ["fc00::/7", "..."],
+ "flowtable_devices": ["wan", "eth1"], "probation_seconds": 120, "manual_confirm": false,
+ "stats_per_device": false}
+```
+
+Lists replace the old ones. `probation_seconds` 0 is the default (120).
+
+### Objects
+
+Zones, sets, services, rules, port forwards, source NAT, redirects and
+ban lists share one pattern, `/{kind}` with kind `zones`, `sets`,
+`services`, `rules`, `forwards`, `snats`, `redirects` or `banlists`:
+
+| | |
+|---|---|
+| `GET /{kind}` | all of them, in order; `GET /rules?chain=input` for one chain; `GET /services` includes the built-in ones |
+| `POST /{kind}[?before=ID\|after=ID\|first=1]` | create; rules, forwards and NAT rules go last unless given a `position`, or placed before or after another of their chain, or first, in the same change (one history entry; staged in a changeset, the place goes with it) |
+| `GET /{kind}/{ref}` | by ID, or name for zones, sets, services, forwards and ban lists |
+| `PATCH /{kind}/{ref}` | change the fields given (a JSON merge: `null` clears) |
+| `DELETE /{kind}/{ref}` | refused while something uses it |
+| `POST /rules/{ref}/reset-quota` | start a rule's quota again now (`config:write`; audited, not staged). Optional body: `{"address": "192.168.1.23"}` starts only that address's again (a per-address quota; an IPv6 address counts for its /64; 404 if it hasn't used the quota); `{"give": 5368709120}` takes that many bytes off what was used instead (the address's, or every address's), until the next reset |
+| `POST /{kind}/{ref}/move` | `{"before": ID}`, `{"after": ID}` or `{"to": "start"\|"end"}`; a rule can take `"chain"` too |
+
+The objects:
+
+```json
+{"name": "lan", "interfaces": ["eth1", "eth1.*"], "external": false, "comment": ""}
+{"name": "trusted", "entries": ["192.168.1.0/24", "2001:db8:1::/64"], "dynamic": false, "timeout": 0, "comment": ""}
+{"name": "nas", "entries": [], "jarl": {"names": [{"name": "nas.example.com", "view": "internal"}], "within": ["lan", "10.9.0.0/16"]}, "comment": ""}
+{"name": "web", "entries": ["tcp/8080", "tcp/8443"], "comment": ""}
+{"chain": "input",
+ "in_zones": ["wan"], "not_in": false, "out_zones": [], "not_out": false,
+ "sources": ["trusted"], "not_sources": false, "destinations": [], "not_destinations": false,
+ "services": ["ssh"], "family": "",
+ "ct_state": ["new"], "ct_status": [], "days": [], "hour_from": "", "hour_to": "",
+ "limit": {"rate": 3, "unit": "", "per": "minute", "burst": 5, "over": true, "per_source": true, "per_destination": false}, "conn_limit": 0,
+ "quota": {"bytes": 53687091200, "over": true, "per_source": true, "per_destination": false, "reset": "monthly", "day": 15},
+ "group_v4": 0, "group_v6": 0,
+ "action": "drop", "reject_with": "", "log": true, "log_prefix": "",
+ "raw": "", "comment": "", "disabled": false}
+{"name": "web", "protocols": ["tcp"], "ports": "8443", "public_addresses": ["203.0.113.10"], "sources": [],
+ "to": "192.168.1.10", "to_port": "443", "to6": "2001:db8:1::10", "hairpin": true, "hairpin_sources": [],
+ "comment": "", "disabled": false}
+```
+
+A set with `jarl` takes its addresses from Jarl ([jarl.md](jarl.md#sets-from-jarl)):
+`names` are names in Jarl, each with a `view` (left out: the only view that
+has it); `within` are zones and addresses its addresses must lie in. It
+has no `entries` and isn't `dynamic`; `"jarl": null` makes it an ordinary
+set again.
+
+A source NAT rule is a match (as a rule's, without the rule's other
+fields) with `"to"` (empty: masquerade), a redirect a match with
+`"to_port"`. A ban list ([bans.md](bans.md)):
+
+```json
+{"name": "web", "scope": "hosts", "hosts": ["10.0.10.57"], "never": ["trusted"],
+ "default_time": 3600, "max_time": 604800, "max_entries": 10000, "group_v6": 64, "comment": "", "disabled": false}
+```
+
+`scope` is `firewall`, `hosts` or `everything`; 0 is the default for the
+numbers, and `max_time` -1 is no limit. A ban list can't be renamed.
+Someone with a `bans` grant but no `config:read` may read the ban lists
+the grant covers (`GET /banlists`, `GET /banlists/{ref}`). Times are
+seconds (`timeout`).
+
+### Statistics
+
+`GET /stats/status`, `/stats/summary`, `/stats/timeline`, `/stats/top`,
+`/stats/hits` and `/stats/heatmap` (`log:read`): see
+[stats.md](stats.md#the-api).
+
+### Changesets
+
+Changes staged to be reviewed and applied together ([design.md](design.md#reviewing-changes-changesets)).
+
+| | |
+|---|---|
+| `GET /changesets[?status=open&mine=1&limit=]` | newest first |
+| `POST /changesets` | `{"description"}`; answers the changeset (201) |
+| `GET /changesets/{id}` | `{"id", "username", "via", "description", "status": "open"\|"applied"\|"discarded", "ops": [{"seq", "op", "kind", "ref", "id", "data", "summary", "added_at"}], "created_at", "updated_at", "history_id"}` |
+| `PATCH /changesets/{id}` | `{"description"}` |
+| `GET /changesets/{id}/preview` | `{"valid", "changes", "warnings", "error", "diff"}`: what applying it would do; `error.op` names an operation that no longer applies |
+| `POST /changesets/{id}/apply` | optional `{"description"}`; answers like a change, with `confirm_by` over the network and `changeset` |
+| `POST /changesets/{id}/discard` | |
+| `DELETE /changesets/{id}/ops/{seq}` | take one operation out |
+
+Staging (`?changeset=ID` on any change to objects or settings) checks
+the changeset with the new operation, and nftables checks its ruleset;
+an error refuses the operation. Only open changesets take operations, and
+only their owner's (or an administrator's). An undo
+(`POST /history/{id}/revert?changeset=ID`) is staged as one operation,
+which checks when it is replayed that nothing it touches changed since;
+applied, History links it to the entry it undoes. `/import/nft` and
+`/takeover` can't be staged: with a changeset named they answer 400.
+
+Reads of objects, settings and `/ruleset` with `?changeset=ID` show the
+configuration as the changeset will make it; objects are marked
+`"pending": "new"`, `"changed"` or `"deleted"` (deleted ones stay in the
+list). A changeset that no longer applies shows the current
+configuration, with the reason in the `X-Housecarl-Changeset-Error`
+header.
+
+### Dynamic sets' entries
+
+| | |
+|---|---|
+| `GET /sets/{name}/elements` | `[{"address", "timeout"}]`, `timeout` the seconds left |
+| `POST /sets/{name}/elements` | `{"address": "10.0.30.17", "timeout": 3600}`; `timeout` 0 is the set's |
+| `DELETE /sets/{name}/elements/{address}` | |
+
+These need `sets:read` or `sets:write`, for that set
+([permissions.md](permissions.md)); they aren't on probation or in the
+history, but are audited.
+
+### Sets from Jarl
+
+| | |
+|---|---|
+| `GET /sets/{name}/jarl` | what the set holds and why (below) |
+| `POST /sets/{name}/jarl/refresh` | reads Jarl now (for every set from Jarl), then answers as `GET` |
+
+They need `sets:read` for that set. 400 for a set that doesn't come from
+Jarl. The answer:
+
+```json
+{"set": "nas", "configured": true, "read": "2026-10-23T14:02:11Z", "error": "",
+ "entries": ["10.10.0.20", "10.10.0.21"], "within": ["lan"],
+ "names": [{"name": "files.example.com", "view": "", "found_view": "internal", "chain": ["nas.example.com"],
+            "addresses": [{"address": "10.10.0.20", "used": true},
+                          {"address": "2001:db8:10::20", "used": false, "why": "not within lan"}],
+            "problem": "", "checked": "2026-10-23T14:02:11Z", "error": ""}]}
+```
+
+* `configured`: `housecarl.yaml` names a Jarl server; `read`: when Jarl's
+  zones were last read; `error`: why the last read failed (the set keeps
+  its addresses).
+* `entries`: the addresses the set holds now.
+* For each name: `found_view` is the view it was read from, `chain` the
+  CNAMEs followed, `addresses` every address Jarl gives it with `used`
+  (or `why` not), `problem` why it gives none, `checked` when Jarl last
+  answered for it, and `error` why the last try failed (its addresses are
+  then those of `checked`).
+
+### Bans
+
+| | |
+|---|---|
+| `GET /bans` | the active bans of every ban list you may read: `id`, `list_id`, `list`, `address` (an address, or an IPv6 prefix), `protocol` and `port` (none: every port), `expires` and `timeout` (seconds left; none: no end), `created`, `who`, `source`, `reason` |
+| `GET /banlists/{ref}/bans` | one list's |
+| `POST /banlists/{ref}/bans` | ban: `{"address": "203.0.113.9", "protocol": "tcp", "ports": "ssh", "time": 3600, "source": "guard/ssh", "reason": "…"}`. `ports` is a number, a string (`"22"`, `"http,https"`, `"0:65535"` or `"all"` for every port) or a list; without a protocol a number is TCP and UDP, a service its own protocols. `time` 0 is the list's default, -1 its longest; longer is cut down. 201 with `{"bans": [ban…, "existing": true for a renewed one], "closed": N}` (connections closed) |
+| `DELETE /banlists/{ref}/bans/{id}` | lift one ban |
+| `DELETE /banlists/{ref}/bans?address=A` | lift all of an address's bans (404 if it has none) |
+
+These need `bans:read` or `bans:write` for the list
+([permissions.md](permissions.md)); bans aren't on probation or in the
+history, but are audited (`ban.add`, `ban.remove`). Refusals: `422
+never_ban` (inside, the firewall's own, the caller's own, or in the
+list's `never`), `422 list_full`, `409` for a disabled list.
+
+### Probation
+
+| | |
+|---|---|
+| `GET /probation` | `{"pending": [history IDs], "takeovers": N, "deadline", "seconds_left"}` |
+| `POST /confirm` | keep them: `{"confirmed": N}`; the ruleset is saved for the boot. With `{"token"}`, the client confirming its own change (see `confirm_token`): 409 if a newer change is pending, or over the change's own connection; 403 from another user or address |
+| `POST /rollback` | undo them now |
+
+### History, import, takeover
+
+| | |
+|---|---|
+| `GET /history[?limit=&before=]` | changes, newest first |
+| `GET /history/{id}` | one, with each object before and after |
+| `GET /history/{id}/diff` | `{"diff"}`: what it did to the ruleset, as a unified diff with each hunk's chain (the configuration before and after it, got back from the current one, rendered by this version) |
+| `POST /history/{id}/revert` | undo it; 409 if an object it touched has changed since, or it was undone already |
+| `POST /import/nft` | `{"json": NFT_JSON, "text": "nft -a list ruleset output", "defines": "ruleset file", "table": "inet main", "zones": {"enp2s0": "lan"}, "replace": false}` ([import.md](import.md)); answers with `state` (the imported configuration), `notes` and `raw` too |
+| `POST /takeover` | `{"tables": ["inet main"]}`: delete these tables (administrators); on probation over the network; 409 in shadow mode |
+
+### Certificate
+
+Administrators only ([certificates.md](certificates.md)).
+
+| | |
+|---|---|
+| `GET /certificate` | `{"configured", "source": "acme"\|"file"\|"none", "status": {"domain", "directory", "staging", "state": "none"\|"ok"\|"working"\|"error", "not_before", "not_after", "issuer", "renew_at", "last_error", "last_attempt", "next_attempt", "check"}}` |
+| `POST /certificate/check` | run the check now (up to a few minutes); answers `{"time", "domain", "ok", "steps": [{"name", "status", "detail", "queries"}]}` |
+| `POST /certificate/renew` | get a new certificate now, in the background (202); follow it with `GET /certificate` |
+
+### Users, tokens, audit
+
+| | |
+|---|---|
+| `POST /auth/login`, `POST /auth/logout`, `GET /auth/whoami` | `whoami` has `token_id` when the caller uses a token |
+| `POST /me/password` | `{"current", "password"}` |
+| `GET /me/tokens`, `POST /me/tokens` | `{"name", "role", "grants", "expires_in"}`; the answer's `secret` is shown once |
+| `GET /tokens`, `DELETE /tokens/{id}` | all tokens (administrators), revoke |
+| `POST /tokens/{id}/rotate` | a new secret: `{"grace": "1h", "expires_in": "90d", "no_expiry": false}`, all optional; answers with `secret` and `previous_secret_valid_until` |
+| `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}` | administrators; `{"username", "password", "admin", "role", "grants", "disabled", "comment"}` |
+| `GET`, `POST /users/{id}/tokens` | |
+| `GET /audit[?user=&action=&result=&q=&before=&limit=]` | |

docs/bans.md

@@ -0,0 +1,148 @@
+# Ban lists
+
+A **ban list** is where a program bans addresses: Guard (Housecarl's
+companion, which reads logs), fail2ban with Housecarl's action, or you
+with `hc ban`. The program says only *which address*, and perhaps a
+protocol and ports. The ban list says what that address is kept from.
+So a web server that sees password guessing in its own logs can ban the
+guesser at the router without knowing anything about the router.
+
+## Ban lists
+
+```sh
+hc banlist add router --scope firewall                   # the firewall itself
+hc banlist add web --scope hosts --host 10.0.10.57 --host 2001:db8:10::57
+hc banlist add edge --scope everything --never trusted
+```
+
+| Scope | Banned addresses are kept from | Where the drops are |
+|---|---|---|
+| `firewall` | the firewall itself: its SSH, DNS, web UI | `input` |
+| `hosts` | the list's hosts (addresses or sets), behind the firewall | `forward`, to those addresses |
+| `everything` | the firewall and everything behind it | `input` and `forward` |
+
+A `hosts` list covers port forwards to its hosts as well: the forward
+chain sees a forwarded connection's translated address and port, the
+host's own. A web server's Guard bans `tcp/22`, and that is right even
+when the router forwards its port 2222 to it.
+
+Other settings, with their defaults:
+
+| Setting | Default | |
+|---|---|---|
+| `never` | none | addresses, prefixes or sets this list never bans |
+| `default_time` (`--time`) | 1h | a ban's length when the program asks for none |
+| `max_time` (`--max-time`) | 7d | the longest ban; longer ones are cut to it. `-1` (`none`): no limit, so bans may be permanent |
+| `max_entries` | 10000 | the most bans the list holds; more are refused (`list_full`) |
+| `group_v6` | 64 | the IPv6 prefix length banned for an address. One customer usually has a whole /64 and can move around in it; 128 bans the one address. IPv4 addresses are banned alone |
+| `disabled` | no | the list's bans aren't in force (they are kept) |
+
+A ban list can't be renamed: the tokens and programs that ban into it
+know it by its name. Make a new one instead.
+
+## Bans
+
+```sh
+hc ban edge 203.0.113.9 --time 1d --reason "port scan"
+hc ban web 203.0.113.9 --tcp ssh --tcp 8443
+hc bans                       # every list's
+hc bans web
+hc unban web 203.0.113.9      # all of the address's bans in web
+hc unban web 17               # one, by ID
+```
+
+A ban is of **every port**, or of **a protocol's ports** (numbers or
+service names, built-in or yours). A port without a protocol is banned
+for TCP and UDP; a service name takes the service's protocols. Banning an
+address again renews the ban: its new time counts from now.
+
+Bans are dropped **ahead of every rule** in their chains, and ahead of
+the built-in that lets established connections through, so a ban cuts a
+connection that is already open. Housecarl also closes the banned
+address's tracked connections the ban covers when it bans it, because
+connections offloaded to the flowtable no longer pass the rules at all.
+Replies to connections the inside opened to a banned address are
+dropped too.
+
+Bans are kept in the database, apart from the configuration (like a
+dynamic set's entries): they aren't in the history, can't be undone,
+aren't on probation, and don't go through changesets. They end by
+themselves; every change, restart and reboot puts the active ones back
+with the time they have left. Each ban and unban is in the audit log
+(`ban.add`, `ban.remove`), with the program's source and reason.
+
+### What is never banned
+
+Whatever a list's `never` says, Housecarl refuses (`422 never_ban`) to
+ban:
+
+* the inside: addresses in networks of inside zones' interfaces, and
+  private addresses outside the external zones' networks;
+* the firewall's own addresses, and its own prefixes (settings);
+* the address of whoever asks (a program can't lock itself out);
+* loopback, link-local, multicast and unspecified addresses.
+
+A ban grouped to a prefix (IPv6) is refused when the prefix overlaps any
+of these.
+
+## Programs and tokens
+
+Give each program a token for its list alone. A `bans` grant can name
+ban lists, as a `sets` grant names sets:
+
+```sh
+hc user add guard-web --no-password --grant bans:write:web
+hc token create web --user guard-web          # as root on the firewall
+```
+
+With it, the program may ban into `web` and lift its bans, list them,
+and read the list itself (`GET /banlists/web`: its grouping and times),
+and nothing else. `viewer` has `bans:read`, `operator` `bans:write`.
+
+## Guard
+
+Guard reads log files and the systemd journal on any machine, and bans
+the addresses that misbehave into a ban list (its own documentation).
+Run it on the web server with a token for `web`, and on the router with
+one for `router` (or the local socket).
+
+## fail2ban
+
+Housecarl ships an action for fail2ban,
+`/usr/share/doc/housecarl/contrib/fail2ban/action.d/housecarl.conf`
+(`contrib/` in the source). Copy it to `/etc/fail2ban/action.d/`, put
+the token in `/etc/fail2ban/housecarl.token` (mode 0600), and use it in a
+jail:
+
+```ini
+[sshd]
+enabled = true
+action  = housecarl[list=web, url="https://router.lan:8070"]
+```
+
+The jail's `port` and `protocol` go with each ban: `port = ssh` bans only
+`tcp/22`, fail2ban's default `0:65535` bans every port. A `bantime` of
+`-1` is the list's longest ban. When fail2ban unbans an address Housecarl
+already let go (its own time ran out first), nothing happens. Add
+`curlopts = --cacert /etc/fail2ban/housecarl-ca.pem` for a private CA.
+
+## Shadow mode
+
+In shadow mode the ban rules count what they would drop
+(`[would drop]`) and let it through, like every other rule, and banning
+closes no connections.
+
+## The ruleset
+
+Each list has four sets, `ban_<id>_v4`, `ban_<id>_v6` (every port) and
+`ban_<id>_v4_svc`, `ban_<id>_v6_svc` (address, protocol and port), all
+with timeouts:
+
+```
+ip saddr @ban_3_v4 counter drop comment "hc:ban3"
+ip saddr . meta l4proto . th dport @ban_3_v4_svc counter drop comment "hc:ban3"
+ip6 saddr and ffff:ffff:ffff:ffff:: @ban_3_v6 counter drop comment "hc:ban3"
+```
+
+Set names starting with `ban_` are therefore Housecarl's own
+(`set_name_reserved`).

docs/certificates.md

@@ -0,0 +1,120 @@
+# The web UI's certificate
+
+Housecarl can serve its web UI and API over HTTPS in two ways:
+
+* **A certificate you manage**: `tls.cert_file` and `tls.key_file` in
+  `housecarl.yaml`. housecarld rereads them when they change
+  ([configuration.md](configuration.md#https)).
+* **A certificate it gets itself** from Let's Encrypt (or another ACME CA),
+  with Jarl publishing the DNS-01 challenge. That's this page.
+
+With DNS-01 the web UI doesn't need to be reachable from the internet.
+`fw.adbraun.lol` can point at a private address and the certificate
+still works on your network. Only the challenge record has to be visible
+on the internet, and Jarl puts it in every view that has your zone.
+
+## Setting it up
+
+1. **Connect Housecarl to Jarl** with a user and a token that may only write
+   Housecarl's own challenge record: [jarl.md](jarl.md).
+2. **Give your web UI a name in DNS**, such as `fw.adbraun.lol` with the
+   Housecarl server's address, in Jarl. It is the name you will type in the
+   browser.
+3. **Configure the certificate** in `/etc/housecarl/housecarl.yaml`:
+
+   ```yaml
+   listen: ["192.168.1.2:8070"]        # an address your browser can reach
+   tls:
+     acme:
+       domain: fw.adbraun.lol
+       email: you@adbraun.lol          # Let's Encrypt writes here if something is wrong
+       directory: letsencrypt-staging  # first; see below
+       agree_tos: true                 # you accept Let's Encrypt's terms of service
+   jarl:
+     server: https://dns.adbraun.lol:8053
+     token_file: /etc/housecarl/jarl.token
+   ```
+
+4. **Restart housecarld** (`sudo systemctl restart housecarl`), and check:
+
+   ```sh
+   sudo hc cert check
+   ```
+
+5. **Watch it get the certificate:** `sudo hc cert`, or Server →
+   Certificate in the web UI. Then open `https://fw.adbraun.lol:8070/`.
+6. **Switch to the real CA.** The staging certificate works, but browsers
+   warn about it, because Let's Encrypt's staging service isn't trusted.
+   It's there to prove everything works without touching Let's Encrypt's
+   rate limits. Once it does, set `directory: letsencrypt`, restart, and
+   Housecarl gets a trusted certificate right away.
+
+Until the first certificate arrives, the web UI is served over plain
+HTTP. housecarld switches to HTTPS on the same port without a restart. From
+then on, plain HTTP is redirected to HTTPS, except from the server itself
+(`127.0.0.1`), so tools there keep working.
+
+## Checking
+
+Before every certificate order, and whenever you ask (`hc cert
+check`, or Test in the web UI), Housecarl checks the whole path without
+asking the CA. Let's Encrypt allows only 5 failed validations per name
+per hour, so a broken setup shouldn't find out the hard way.
+
+| Step | What it checks |
+|---|---|
+| Jarl | Jarl answers at `jarl.server`, and the token is valid (it says whose it is) |
+| Zone | a primary zone the token can see holds `_acme-challenge.<domain>`, and in which views |
+| Permission | the token may write the TXT record there (Jarl checks it as a dry run) |
+| Write | a test record with a random value is written through Jarl, which loads it into named |
+| Name servers | each of the zone's public name servers serves it, asked directly as the CA's resolvers will. A name server passes if any of its addresses does (`tls.acme.check_propagation`) |
+| Public resolvers | the resolvers in `tls.acme.resolvers` (1.1.1.1, 9.9.9.9, 8.8.8.8) see it. If the name servers do but a resolver doesn't, it is only a warning: it may be holding an earlier answer |
+| Clean up | the test record is removed |
+
+Each step says what went wrong and what to do. For example:
+
+```
+  ok    Jarl        the token is housecarl's, on Jarl v0.6.1
+  ok    Zone        adbraun.lol in views internal, external
+  FAIL  Permission  the token may not write the challenge record (...); give it the grant:
+                    records:write zones=adbraun.lol names=_acme-challenge.fw.adbraun.lol types=TXT
+```
+
+The checks wait up to 90 seconds for the record to appear. A failing step
+lists the DNS questions it asked, as `dig` commands, so you can ask them
+yourself.
+
+If the challenge name is a CNAME into another zone, the check follows it,
+as Let's Encrypt does.
+
+## Renewal
+
+A certificate is renewed when a third of its lifetime is left: 30 days
+before the end for Let's Encrypt's 90-day certificates, 2 days for 6-day
+ones. housecarld looks twice a day. The check runs first every time, and
+the new certificate is used at once, without a restart.
+
+If renewing fails, the old certificate keeps being served, and housecarld
+tries again after 1 hour, then 3, 6 and every 12. The dashboard shows a
+problem (`certificate_failed`), and `certificate_expiring` once less than
+14 days are left. `hc cert renew` (Renew now in the web UI) tries
+again at once.
+
+The ACME account and the certificate, with its key, are kept in Housecarl's
+database, so backups include them.
+
+## Settings
+
+All under `tls.acme` in `housecarl.yaml` ([configuration.md](configuration.md)):
+
+| Setting | Default | |
+|---|---|---|
+| `domain` | none (off) | the name on the certificate |
+| `email` | none | the account's contact address |
+| `directory` | `letsencrypt-staging` | `letsencrypt`, `letsencrypt-staging`, or an ACME directory URL (a private CA such as step-ca) |
+| `agree_tos` | `false` | accept the CA's terms of service; no account is made without it |
+| `resolvers` | `1.1.1.1:53`, `9.9.9.9:53`, `8.8.8.8:53` | public resolvers, to see the challenge as the CA does |
+| `check_propagation` | `true` | also ask each of the zone's name servers directly |
+| `ca_bundle` | none | extra CAs to trust for the ACME server itself |
+
+`tls.cert_file` and `tls.acme` can't both be set.

docs/cli.md

@@ -0,0 +1,282 @@
+# The hc CLI
+
+`hc` talks to housecarld's REST API. `hc help COMMAND` describes each
+command and its flags.
+
+## Connecting
+
+* **On the firewall**, with nothing configured, the CLI uses
+  housecarld's local socket, `/run/housecarl/housecarl.sock`. Root and
+  the `housecarl` user are administrators there without a token, and
+  their changes need no confirmation: `sudo hc status`.
+* **From elsewhere**, give the server's URL and an API token:
+
+  ```sh
+  hc config set server https://fw.example.com:8070
+  hc config set token          # asks for it
+  ```
+
+  The settings are in `~/.config/housecarl/config.yaml` (on macOS
+  `~/Library/Application Support/housecarl/config.yaml`, on Windows
+  `%AppData%\housecarl\config.yaml`).
+
+| Flag | Environment | |
+|---|---|---|
+| `-s, --server URL` | `HOUSECARL_SERVER` | the server; `local` for the local socket |
+| `--token TOKEN` or `@FILE` | `HOUSECARL_TOKEN` | the API token |
+| `--config FILE` | `HOUSECARL_CLI_CONFIG` | the CLI's configuration file |
+| | `HOUSECARL_SOCKET` | the local socket's path |
+| `--insecure`, `--ca-cert FILE` | | TLS: skip the check, or trust another CA |
+| `-n, --dry-run` | | check and show a change without making it, with the ruleset's diff |
+| `--no-confirm` | | leave a change on probation instead of confirming it once `hc` reaches Housecarl again |
+| `-C, --changeset ID` | `HOUSECARL_CHANGESET` (checked, see [Changesets](#changesets)) | stage configuration changes in this changeset instead of making them |
+| `-o, --output FORMAT` | | `table` (default), `json`, `yaml` or `id` |
+| `-q, --quiet` | | print only errors (and the IDs of new objects) |
+| `--no-names` | | show inside addresses without their names: tables show `192.168.1.20 (laptop)` ([configuration.md](configuration.md#names)) |
+
+Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not
+found, 5 conflict, 6 invalid or refused by validation or nftables, 7
+server error.
+
+## Changes
+
+Every change is checked as a whole, then by nftables, before it is put
+in force. A refused change says why, with the problem's code
+([validation.md](validation.md)). An applied one says its history entry
+and any new warnings. Over the network, it is on probation, and `hc`
+confirms it by reaching Housecarl again over a new connection
+([operations.md](operations.md#changes-on-probation)):
+
+```
+$ hc rule add input --from trusted --service ssh --comment "SSH from home"
+created rule input #31 (from trusted · ssh)
+applied and confirmed (history 12): this client reached Housecarl again over a new connection
+```
+
+Through the socket the change is kept at once:
+
+```
+applied and saved (history 12; undo with 'hc history undo 12')
+```
+
+`-n` shows what would change without changing it, with the diff of the
+ruleset (each `@@` line names the chain the lines below it are in);
+nftables checks it too.
+
+## Changesets
+
+To review several changes and apply them together, as one entry in the
+history, stage them in a changeset:
+
+```sh
+export HOUSECARL_CHANGESET=$(hc changeset create "guest network" -q)
+hc zone add guest -i eth1.30
+hc rule add forward --in guest --out wan
+hc changeset diff                 # the operations, problems, nftables' check, the ruleset's diff
+hc changeset remove 2             # take operation 2 out
+hc changeset apply                # one change; over the network on probation
+unset HOUSECARL_CHANGESET
+```
+
+```
+$ hc rule add forward --in guest --out wan
+changeset 7 (guest network, 1 operation) from HOUSECARL_CHANGESET
+staged: create rule forward #52
+staged in changeset 7 as operation 2 (2 operations now); review with 'hc changeset diff 7', apply with 'hc changeset apply 7'
+```
+
+Each change is checked as it is staged: the changeset with it must stay
+valid, and nftables must accept the ruleset, so make a set before the
+rule that names it. Applying checks the whole changeset again, on the
+configuration as it is then; an operation whose object changed since is
+named (`operation 3 (update rule input #12) can't be applied: …`), and
+`hc changeset remove` takes it out.
+
+| Command | |
+|---|---|
+| `changeset create [DESCRIPTION]` | start one; `-q` prints only its ID |
+| `changeset list [--status open\|applied\|discarded\|""] [--mine]` | open ones by default |
+| `changeset show [ID]` | its operations |
+| `changeset diff [ID]` | what applying it would do |
+| `changeset apply [ID] [--description D]` | apply it |
+| `changeset discard [ID]` | throw it away |
+| `changeset remove [ID] OP` | take one operation out |
+
+`show`, `diff`, `apply`, `discard` and `remove` work on the changeset you
+name, else the one in `-C`, else the one in `HOUSECARL_CHANGESET`.
+
+A changeset from `HOUSECARL_CHANGESET` is checked before anything uses
+it, because an exported ID easily outlives its changeset:
+
+- Staging into it, and `apply`, `discard` and `remove` on it, need it to
+  be open and yours. Otherwise the command stops and says why (`there is
+  no changeset 99`, `changeset 42 is applied`, `changeset 57 is sam's`).
+  Commands that only read, and `hc changeset create`, don't care.
+- Each command says on stderr which changeset it uses: `changeset 42
+  (guest network, 2 operations) from HOUSECARL_CHANGESET` (not with `-q`).
+- After `apply` or `discard`, a reminder to `unset HOUSECARL_CHANGESET`.
+
+`-C ID` and an ID argument are used as given (housecarld still refuses a
+changeset that isn't open, or another user's unless you are an
+administrator). Only configuration changes are staged: dynamic set
+entries, `confirm` and `rollback` are made at once; `history undo` is
+staged like any change; `import` and `takeover` can't be staged, so they
+are refused while a changeset is named. Changesets staged in the web UI show up here too,
+and the other way round.
+
+## Commands
+
+Objects are named by ID or, for zones, sets, services and port forwards,
+by name.
+
+### status and the ruleset
+
+| Command | |
+|---|---|
+| `status` | the mode (in shadow mode), the ruleset (in force or not), changes on probation, problems, interfaces, other tables |
+| `interfaces` | the machine's interfaces, their addresses and zones |
+| `ruleset [--enforcing]` | the nftables ruleset Housecarl builds; in shadow mode, `--enforcing` shows the one loaded once it is off ([shadow.md](shadow.md)) |
+| `counters` | the hits of every rule, NAT rule, built-in and policy, by mark (`r12`, `f3`, `b-established`, `p-input`) |
+| `quotas` | what each rule's quota used (each address's, for per-address quotas), and when it resets |
+| `log [-f] [--search TEXT] [--limit N]` | the packets logged by rules and policies ([operations.md](operations.md#the-log)) |
+| `connections [--search TEXT] [--limit N]`, `connections close --search TEXT [-y]` | tracked connections; closing makes the rules check them again ([operations.md](operations.md#connections)); not in shadow mode |
+| `stats [--since 24h]` | traffic per zone, drops, the busiest rules, blocked sources, devices; `stats status`, `stats traffic`, `stats top devices\|blocked\|tried\|quota ID`, `stats rules [--unused]`, `stats connections` ([stats.md](stats.md#the-cli)) |
+| `matrix [FROM TO]` | what the rules do to new connections between zones (allow, block or partly), and what is let through; with FROM and TO, that pair's rules in order, with their hits ([rules.md](rules.md#the-zone-matrix)) |
+| `trace [--address A] [--port N] [--protocol P] [--seconds S]` | follow packets through every table's rules ([operations.md](operations.md#tracing)) |
+
+### zone, set, service
+
+| Command | |
+|---|---|
+| `zone list`, `zone show ZONE` | |
+| `zone add NAME -i IFACE... [--external] [--comment C]` | |
+| `zone update ZONE [-i IFACE]... [--add IFACE] [--remove IFACE] [--external[=false]] [--name N] [--comment C]` | `-i` replaces the interfaces; `--name` renames it, and the rules naming it follow |
+| `zone delete ZONE` | refused while rules use it |
+| `set list`, `set show SET` | |
+| `set add NAME [ENTRY...] [--dynamic] [--timeout 24h] [--comment C]` | |
+| `set add NAME --jarl NAME[@VIEW]... --within ZONE\|NETWORK...` | a set from Jarl: the names' addresses, only those within the zones and networks ([jarl.md](jarl.md#sets-from-jarl)) |
+| `set update SET [--entry E]... [--add E] [--remove E] [--timeout] [--jarl N]... [--within W]... [--name] [--comment]` | `--entry` replaces the entries, `--jarl` a Jarl set's names, `--within` where its addresses may be |
+| `set entries SET`, `set entries add SET ADDRESS [--timeout 1h]`, `set entries remove SET ADDRESS` | a dynamic set's entries |
+| `set jarl SET [--refresh]` | a set from Jarl: each name's addresses in Jarl (CNAMEs followed), used or why not, and when Jarl answered; `--refresh` reads Jarl first |
+| `set delete SET` | |
+| `service list`, `service show SERVICE` | with the built-in ones |
+| `service add NAME ENTRY... [--comment C]` | entries `tcp/22`, `udp/1000-2000`, `icmp/echo-request`, `icmpv6/packet-too-big` |
+| `service update SERVICE`, `service delete SERVICE` | |
+
+See [rules.md](rules.md).
+
+### rule
+
+| Command | |
+|---|---|
+| `rule list [CHAIN]` | in order, with their hits |
+| `rule show ID` | |
+| `rule add CHAIN [MATCH] [-a ACTION] [--first\|--before ID\|--after ID]` | at the end of its chain unless placed |
+| `rule update ID [flags]` | only the flags given |
+| `rule move ID --before ID\|--after ID\|--first\|--last\|--chain CHAIN` | |
+| `rule enable\|disable ID` | |
+| `rule reset-quota ID [--address ADDR] [--give 5gbytes]` | start a rule's quota again now, or one address's; `--give` takes an amount off what was used instead, until the next reset |
+| `rule delete ID` | |
+
+The match flags (`rule add`, `snat add`, `redirect add`):
+
+| | |
+|---|---|
+| `--in ZONE`, `--out ZONE` (`--not-in`, `--not-out`) | zones |
+| `--from`, `--to` (`--not-from`, `--not-to`) | sets, addresses, prefixes, ranges |
+| `--service` | services or entries |
+| `--family ipv4\|ipv6` | |
+
+And for rules: `--state`, `--status`, `--days`, `--hours`, `--limit
+10/minute|1mbytes/second[,burst=N][,over][,per-source|,per-destination]`,
+`--quota 50gbytes[,over][,per-source|,per-destination][,daily|,weekly|,monthly=DAY]`,
+`--group-v4 N`, `--group-v6 N`, `--conn-limit N`, `--every-packet`, `-a
+accept|drop|reject|log`, `--reject-with tcp-reset`, `--log`,
+`--log-prefix`, `--raw RULE`, `--comment`, `--disabled`
+([rules.md](rules.md#rules)).
+
+### forward, snat, redirect
+
+| Command | |
+|---|---|
+| `forward list`, `forward show FORWARD` | port forwards, with their hits |
+| `forward add PORTS --to HOST[:PORT] [--proto tcp,udp] [--public ADDR]... [--from SRC]... [--hairpin] [--hairpin-from SRC]... [--to6 HOST] [--name N] [--comment C]` | |
+| `forward update FORWARD [flags]`, `forward delete FORWARD` | |
+| `snat list`, `snat add [MATCH] [--snat-to ADDR]`, `snat update ID`, `snat delete ID` | source NAT, in order; masquerade without `--snat-to` |
+| `redirect list`, `redirect add [MATCH] [--to-port PORT]`, `redirect update ID`, `redirect delete ID` | |
+
+See [nat.md](nat.md).
+
+### banlist, ban
+
+| Command | |
+|---|---|
+| `banlist list`, `banlist show BANLIST` | ban lists, with their bans' count |
+| `banlist add NAME --scope firewall\|hosts\|everything [--host ADDR\|SET]... [--never ADDR\|SET]... [--time 1h] [--max-time 7d\|none] [--max-entries N] [--group-v6 64] [--disabled] [--comment C]` | |
+| `banlist update BANLIST [flags]`, `banlist delete BANLIST` | `--host` and `--never` replace the lists; no renaming |
+| `bans [BANLIST]` | the active bans |
+| `ban BANLIST ADDRESS [--tcp PORT\|SERVICE]... [--udp …]... [--time 1h\|none] [--reason R]` | ban every port, or some; again renews |
+| `unban BANLIST ADDRESS\|ID` | lift one ban, or all of an address's |
+
+See [bans.md](bans.md).
+
+### settings
+
+| Command | |
+|---|---|
+| `settings show` | |
+| `settings set [--input-policy\|--forward-policy\|--output-policy accept\|drop] [--log-drops on\|off] [--builtin NAME=on\|off]... [--own-prefix P]... [--antispoof-v4 P]... [--antispoof-v6 P]... [--flowtable DEV]... [--probation 2m] [--auto-confirm on\|off] [--stats-per-device on\|off]` | lists given replace the old ones; `none` empties one (`--flowtable none`: off) |
+
+### probation, history, import
+
+| Command | |
+|---|---|
+| `confirm` | keep the changes on probation; the ruleset is saved for the boot |
+| `rollback` | undo the changes on probation now |
+| `history list [--limit N]` | changes, newest first |
+| `history show ID` | each object before and after (a changed setting by its own fields: `builtins.drop_invalid: false -> true`) |
+| `history diff ID` | what the change did to the ruleset |
+| `history undo ID` | apply the change backwards (refused if something it touched has changed since) |
+| `import [FILE\|-] [--text FILE] [--defines FILE] [--table T] [--zone IFACE=NAME]... [--replace]` | [import.md](import.md) |
+| `takeover TABLE... [-y]` | delete tables Housecarl's ruleset replaces (administrators); not in shadow mode |
+
+### cert
+
+Administrators only ([certificates.md](certificates.md)).
+
+| Command | |
+|---|---|
+| `cert` | the certificate: name, CA, validity, renewal, the last check |
+| `cert check` | check that the challenge can be published through Jarl and seen, without the CA (exit code 1 if it can't) |
+| `cert renew [-w]` | get a new certificate now; `-w` waits and shows the result |
+
+### users, tokens, audit
+
+| Command | |
+|---|---|
+| `whoami` | who you are to the server, and what you may do |
+| `user list\|add\|update\|delete` | users (administrators); `user add NAME --role operator` |
+| `token list [--all]`, `token create NAME [--role R] [--grant G]... [--expires 90d] [--user U] [--save]`, `token revoke ID` | API tokens |
+| `token rotate ID\|--current [--grace 1h] [--expires 90d\|never] [--save]` | a new secret for a token, the old one working for the grace period |
+| `password` | change your password |
+| `audit [--user U] [--action PREFIX] [--search TEXT]` | the audit log |
+| `config show`, `config set server\|token\|insecure\|ca_cert [VALUE]` | this CLI's settings |
+
+[permissions.md](permissions.md) explains roles and grants.
+
+## housecarld
+
+The daemon's own commands work on the database directly, as root or the
+`housecarl` user (`sudo -u housecarl housecarld ...`):
+
+| Command | |
+|---|---|
+| `housecarld serve` | run (the systemd unit does) |
+| `housecarld init [-u NAME] [--token]` | create the database and the first administrator (asks for the name on a terminal) |
+| `housecarld user add NAME [--admin]`, `user password NAME`, `user token NAME TOKEN-NAME` | when the API can't be used |
+| `housecarld check` | check the configuration against this machine's interfaces |
+| `housecarld backup [FILE]`, `housecarld restore FILE` | [operations.md](operations.md) |
+| `housecarld config [--example]` | the settings in use, or a commented example |
+| `housecarld version` | |
+
+All take `-c FILE` for another configuration file.

docs/configuration.md

@@ -0,0 +1,85 @@
+# Configuration file
+
+`/etc/housecarl/housecarl.yaml` says how housecarld runs. The firewall's
+configuration (zones, sets, rules, NAT, settings) isn't here; it is in
+the database, changed with the CLI, the API or the web UI.
+
+`housecarld config --example` prints a commented file with the defaults.
+Every setting is optional; unknown settings are an error. housecarld
+reads the file when it starts.
+
+| Setting | Default | |
+|---|---|---|
+| `listen` | `["127.0.0.1:8070"]` | addresses of the web UI and API (`host:port`). Keep them on localhost or a management network, and write an input rule that lets you reach them |
+| `tls.cert_file`, `tls.key_file` | none | serve HTTPS with this certificate; housecarld rereads it when the file changes, or on `systemctl reload housecarl` |
+| `tls.acme.*` | off | get the certificate from Let's Encrypt, with Jarl publishing the challenge ([certificates.md](certificates.md#settings)) |
+| `jarl.server`, `jarl.token_file` (or `jarl.token`), `jarl.ca_cert`, `jarl.insecure` | none | the Jarl server for the certificate's challenge and the sets from Jarl, and its token ([jarl.md](jarl.md)) |
+| `jarl.refresh` | `1m` | how often the sets from Jarl read their names (10s or more; [jarl.md](jarl.md#when-the-addresses-change)) |
+| `ruleset` | `/var/lib/housecarl/ruleset.nft` | the confirmed ruleset, which `housecarl-ruleset.service` loads at boot; its fallback is saved beside it as `ruleset-fallback.nft` ([operations.md](operations.md#if-the-ruleset-fails-to-load-at-boot)). Change both paths in the unit too |
+| `trusted_proxies` | `[]` | addresses or prefixes of reverse proxies whose `X-Forwarded-For` and `X-Forwarded-Proto` are believed |
+| `socket` | `/run/housecarl/housecarl.sock` | the local API socket; `""` turns it off |
+| `database` | `/var/lib/housecarl/housecarl.db` | the SQLite database: configuration, history, users, audit log |
+| `log.level` | `info` | `debug` (every HTTP request), `info`, `warn` or `error` |
+| `log.format` | `text` | `text` or `json` |
+| `session.lifetime` | `12h` | how long a web UI login lasts |
+| `session.idle_timeout` | `2h` | logged out after this long without use |
+| `session.cookie_secure` | `auto` | `auto` (secure over HTTPS), `always` or `never` |
+| `backup.dir` | `/var/backups/housecarl` | where `housecarld backup` writes |
+| `backup.retention` | `30d` | backups older than this are deleted… |
+| `backup.keep_at_least` | `7` | …keeping at least this many |
+| `stats.enabled` | `true` | keep traffic, drops, hits, blocked sources, connections and quotas over time ([stats.md](stats.md)) |
+| `stats.database` | `/var/lib/housecarl/stats.db` | the statistics' own database; deleting it is safe |
+| `stats.retention.minute`, `.hour`, `.day` | `7d`, `90d`, `730d` | how long per-minute, hourly and daily figures are kept |
+| `stats.top_n` | `100` | entries kept per top list, per hour and per day |
+| `shadow.enabled` | `false` | shadow mode: Housecarl's ruleset is loaded beside the firewall you have, but blocks, rejects and translates nothing; its counters, log and tracer show what it would do. No change is on probation, and takeovers and closing connections are refused ([shadow.md](shadow.md)) |
+| `shadow.position` | `after` | where Housecarl's filter chains run in shadow mode: `after` the other firewall's (`filter + 10`: it sees only what that one lets through, so what it would drop is what switching would break) or `before` them (`filter - 10`: it sees every packet) |
+| `names.enabled` | `true` | show inside addresses' names beside them, found by reverse lookups (below) |
+| `names.resolver` | `""` | the DNS server to ask, such as `10.10.0.1:53`; empty: the system's (`/etc/hosts`, then `/etc/resolv.conf`) |
+| `names.cache` | `10m` | how long a name, or the lack of one, is kept |
+
+Durations are written `15s`, `12h` or `30d`.
+
+Shadow mode is here, and not a firewall setting, so that no user or API
+token can switch enforcement off; restart housecarld to switch.
+
+### Names
+
+The web UI and `hc` show a name beside each **inside** address in
+connections, the log, quotas and traffic per device: `laptop
+192.168.1.20`. Housecarl finds it with a reverse (PTR) lookup, so the
+names are your reverse zones' (and `/etc/hosts`'). An address counts as
+inside when it is in a network connected to an inside zone's interface,
+or private (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`)
+and not in an external zone's network. Outside addresses get no name:
+their reverse zones belong to whoever holds the addresses, who would
+choose the names, and looking up every scanner is slow.
+
+Names are looked up when a page or `hc` asks, cached (a failed lookup is
+tried again after a minute), and never stored: statistics stay by
+address. With a split DNS (views), the server asked must answer the
+firewall as an inside client; set `names.resolver` if the system's
+resolver doesn't. IPv6 traffic per device is counted per /64, which has
+no name. `hc --no-names` leaves them out.
+
+The probation's length is a firewall setting, not here: `hc settings set
+--probation 5m`. So is counting traffic per device for the statistics:
+`hc settings set --stats-per-device on`.
+
+## HTTPS
+
+Over plain HTTP, passwords and tokens cross the network in the clear;
+housecarld warns when it listens on a non-loopback address without TLS.
+Let it get its own certificate through Jarl ([certificates.md](certificates.md)),
+or give it one:
+
+```yaml
+listen: ["192.168.1.1:8070"]
+tls:
+  cert_file: /etc/housecarl/tls/fullchain.pem
+  key_file: /etc/housecarl/tls/privkey.pem
+```
+
+Or keep it on `127.0.0.1` behind a reverse proxy that terminates TLS, and
+list the proxy in `trusted_proxies`. With a certificate, the same port
+answers HTTPS and redirects plain HTTP to it, except from the firewall
+itself.

docs/design.md

@@ -8,79 +8,213 @@ talks to that API; SQLite for everything. As Jarl does for BIND, it keeps
 a Linux machine's **nftables** ruleset: the database is the source of
 truth, and the ruleset is built from it.
 
-This page is the starting sketch. Nothing here is built yet; it records
-what was decided and what is still open.
+## The model
+
+Named objects, and ordered rules that refer to them
+([rules.md](rules.md), [nat.md](nat.md)):
+
+* **Zones**: interfaces grouped by what they connect to (`wan`, `lan`,
+  `wg`). External zones face the internet.
+* **Address sets**: IPv4 and IPv6 addresses, prefixes and ranges, mixed.
+  A **dynamic** set takes entries at run time, each with a timeout that
+  nftables enforces by itself.
+* **Services**: ports of a protocol and ICMP types, by name. Many are
+  built in (`ssh`, `dns`, `https`, `wireguard`).
+* **Rules**, in order, in four chains: `input` (to the firewall),
+  `forward` (through it), `output` (from it) and `early` (before
+  connection tracking).
+* **NAT**: port forwards (with hairpin NAT and an optional IPv6 pinhole),
+  source NAT (masquerade or an address), redirects to the firewall.
+* **Settings**: the chains' policies, and the **built-ins**, the rules
+  every firewall needs (loopback, established connections, ICMPv6,
+  antispoofing, MSS clamping), each of which can be turned off.
+
+IPv4 and IPv6 are handled together: a set holds both, and a rule that
+names one becomes one or two nftables rules as needed.
 
 ## How a change is made
 
-As in Jarl, every change goes through one engine:
+Every change goes through one engine (`internal/engine`):
 
 1. the request becomes a change to a copy of the configuration;
-2. the whole result is validated, with errors that say what is wrong and
-   how to fix it;
-3. the ruleset is rendered, and checked with `nft -c -f` (a dry run by
-   nftables itself);
+2. the whole result is validated ([validation.md](validation.md)), with
+   errors that say what is wrong and how to fix it;
+3. the ruleset is rendered (`internal/render`) and checked with
+   `nft -c -f` (a dry run by nftables itself);
 4. it is applied with `nft -f`, which replaces Housecarl's table in one
-   transaction: the old rules stay in force until the new ones are all in;
-5. the change, its history entry and the audit record are saved together.
+   transaction: the old rules stay in force until the new ones are all
+   in;
+5. the objects, the history entry and the audit record are saved
+   together. If saving fails, the previous ruleset is applied again.
 
 Dry runs, history with undo, users, sessions and scoped API tokens come
-from the same machinery, as in Jarl and Steward.
+from the same machinery as in Jarl and Steward.
 
-### Not locking yourself out
+## Reviewing changes: changesets
 
-A firewall managed over the network can cut off its own manager. Housecarl
-should guard against that from the start:
+As in Jarl, a change can be made at once or staged in a **changeset**,
+to be reviewed and applied together with others:
 
-* **Confirm or roll back.** A risky change (one touching the rules for the
-  machine itself) is applied on probation: unless it is confirmed within a
-  minute or two, from the web UI or `hc confirm`, the previous ruleset comes
-  back by itself. Network equipment calls this "commit confirmed".
-* **A management rule that can't be dropped by accident**: the API and SSH
-  stay reachable from the management network unless that rule is removed
-  on purpose.
+* Each change is an **operation** (create, update, delete or move an
+  object, or change the settings), kept as the API received it.
+* Staged in a changeset, an operation is checked as it is added: the
+  changeset with it must still be valid, and nftables must accept the
+  ruleset it makes. Nothing is applied.
+* The **review** shows the operations, the problems, and the **diff of
+  the ruleset**: the file that is loaded at boot, now and after.
+* **Applying** replays the operations on the configuration as it is then,
+  as one change: one check, one transaction, one history entry, undone as
+  a whole. An operation that no longer applies (its object was changed or
+  deleted since) is named, and can be taken out.
+* Then the applied changeset is **on probation** like any change made over
+  the network: unless it is confirmed, it is rolled back.
+
+| Interface | Default | Why |
+|---|---|---|
+| Web UI | Review edits: they are collected in a draft changeset, pages show pending rows (green new, yellow changed, struck out deleted), and a bar leads to the Changes page, with the diff and Apply. The switch in the header turns on Live edits, applied at once. | Several related edits, seen together before they are in force. |
+| CLI | Immediate: each command is a change of its own. Stage with `-C ID` or `HOUSECARL_CHANGESET`. | Scripts want a clear result per step. |
+| API | Immediate unless a changeset is named (`?changeset=ID`, `X-Housecarl-Changeset`). | As the CLI. |
+
+An undo is staged like any edit (in review mode, the web UI's Undo
+collects it for review). Imports and takeovers can't be staged; with a
+changeset named, they are refused rather than applied.
 
 ## Its own table only
 
-Housecarl writes one nftables table of its own (`inet housecarl`) and
-leaves every other table alone, such as Docker's or libvirt's. It never
-flushes the whole ruleset.
+Housecarl writes one table, `inet housecarl`, and leaves every other
+table alone (Docker's, Incus's, libvirt's). It never flushes the whole
+ruleset. The table is replaced in one script:
 
-## Concepts (draft)
+```
+table inet housecarl {}
+delete table inet housecarl
+table inet housecarl { ... }
+```
 
-* **Zones**: interfaces or VLANs grouped by trust (`lan`, `iot`, `guest`,
-  `wan`), and the default policy between each pair.
-* **Address sets**: named sets of addresses or prefixes, which rules refer
-  to. An entry may have a **timeout**, which nftables enforces by itself, so
-  an entry expires even if housecarld is down. Other programs can be given
-  a token that may only change one set.
-* **Rules** between zones: allow or drop, by set, port and protocol, with
-  logging and counters.
-* **NAT**: masquerading out of the WAN zone, and port forwards.
-* **Services**: names for ports (`ssh`, `https`, `dns`).
+Two things survive the replacement: the **entries of dynamic sets**
+(read before, written back with their remaining timeouts, in the same
+script), and the **counters** (housecarld adds the old totals to the
+new, so hits don't reset with every change). Sets from Jarl are written
+with the addresses last read from Jarl.
+
+Every rule carries a comment `hc:<mark>`, such as `hc:r12` for rule 12,
+`hc:f3` for port forward 3, `hc:b-established` for a built-in. Counters
+and log lines are traced back to their object through it.
+
+## Shadow mode
+
+With `shadow.enabled` in the configuration file ([shadow.md](shadow.md)),
+the same configuration is rendered so that it changes nothing (render's
+`Options.Shadow`): drops and rejects accept instead, policies accept,
+NAT only counts, and what alters packets (MSS clamping, the flowtable)
+and raw rules that do more than accept, drop or reject are left out.
+This rests on how nftables runs base chains: an `accept` only ends its
+own chain, and the packet goes on to the next one on the hook, other
+tables' included; only a drop is final. Another table, the real
+firewall, decides.
+
+What would be dropped, rejected or translated is counted under the same
+marks, and its comment says so (`"hc:r12 [would drop] …"`), which is
+how the tracer tells it apart. The filter chains run at `filter + 10`
+(after the other firewall's: what Housecarl would drop is what switching
+over would break) or `- 10`; the NAT chains always at `- 10`, since the
+kernel asks NAT chains only until one translates.
+
+Every change is checked by nftables in both forms, so leaving shadow
+mode can't fail on the ruleset. There is no probation, since nothing
+can lock anyone out; takeovers and closing connections, which would
+change traffic, are refused. A ruleset reload elsewhere (`flush
+ruleset`) deletes Housecarl's table too: housecarld listens to nftables'
+netlink notifications (internal/tablewatch) and puts it back.
+
+## Not locking yourself out
+
+A change made over the network is applied **on probation** (a changeset
+when it is applied, a live edit at once): unless it is
+confirmed within two minutes (Settings; `hc confirm`, the banner in the
+web UI, `POST /confirm`), it is rolled back by itself, and the rollback is
+recorded in the history. A change that cuts you off undoes itself.
+
+* **The client that made a change confirms it by itself** once it
+  reaches the API again: the change's answer carries a one-time token and
+  closes its connection (HTTP/2: a GOAWAY), and the web UI and `hc` send
+  the token straight back. That proves something only on a new
+  connection, since the built-in that accepts established connections
+  would let the old one through whatever the change did. So the token is
+  accepted only from the same user (and API token), the same address, a
+  different connection, and for the newest change. A change that cuts its
+  client off can't be confirmed by it, and rolls back. This proves the
+  API is still reachable, not that SSH or the LAN still work; with the
+  API, you can fix those. Settings → "a person confirms each change"
+  (`hc settings set --auto-confirm off`) goes back to confirming by hand;
+  `hc --no-confirm` leaves one change waiting.
+* Changes made through the local socket (`sudo hc …` on the firewall)
+  need no confirmation: whoever makes them is already on the machine. One
+  made while others are pending joins them.
+* A new change restarts the clock for everything pending.
+* If housecarld stops with changes pending, it rolls them back when it
+  starts.
+* Housecarl doesn't force a rule that keeps SSH or the web UI open. You
+  write the rules that let you in; probation is the safety net.
+
+## At boot
+
+The ruleset is saved to `/var/lib/housecarl/ruleset.nft` whenever it is
+confirmed (or changed through the socket). `housecarl-ruleset.service`
+loads it before the network comes up, without housecarld, so the
+firewall is up even if housecarld can't start. An unconfirmed ruleset is
+never saved there: a reboot during probation brings back the last
+confirmed one.
+
+It fails closed. A fallback saved beside it (`ruleset-fallback.nft`,
+`render.Fallback`) is loaded if it doesn't: no sets, no flowtable,
+interfaces by name only, so there is little in it to fail. It keeps
+anything new from the external zones out and lets the inside work,
+which is as far as a ruleset can go without knowing the rules; a
+fallback that let everything in would leave a router open on the
+internet, and one that let nothing in would lock out the person who has
+to fix it. The boot ruleset has no flowtable either: nftables refuses a
+flowtable naming a device that doesn't exist yet (on Debian 13's
+kernel), and before the network is up, `ppp0` doesn't. housecarld
+watches the kernel's link messages (internal/linkwatch) and applies
+again when a flowtable device appears, or comes back as a new interface.
+
+## The drop log
+
+Rules with logging on, and the policies (Settings), send packets to
+netlink log group 4171. housecarld listens there and keeps the last 2000
+in memory (`hc log`, the Log page). Nothing is written to the kernel log.
+
+## Importing a ruleset
+
+`hc import` turns an existing nftables ruleset into zones, sets,
+services, rules and NAT, and `hc takeover` deletes the old table once
+Housecarl's replaces it ([import.md](import.md)).
 
 ## Working with Steward and Jarl
 
-* **Steward's captive portal** (parked until Housecarl exists): a guest
-  network's zone lets clients that haven't signed in reach only DNS and
-  the portal, and sends their web traffic to it. When a guest signs in,
-  Steward adds its address to a set (`guest-signed-in`) for the session's
-  length, with a token that may only change that set. Addresses, not MAC
+* **Steward's captive portal** (parked until now): a guest network's zone
+  lets clients that haven't signed in reach only DNS and the portal, and
+  sends their web traffic to it. When a guest signs in, Steward adds its
+  address to a dynamic set for the session's length, with a token that
+  may only change that set (`sets:write:guests`). Addresses, not MAC
   addresses: matching by MAC only works on the guests' own network
   segment.
-* **Names instead of addresses**: Steward knows every reservation's and
-  lease's address, so rules could name a device ("the printer") or a
-  subnet; Jarl knows the names in DNS.
-* **Steward's deny lists** could also be enforced here, so a denied device
-  with an address set by hand is still blocked.
+* **Names instead of addresses**: a set can take its addresses from Jarl
+  ([jarl.md](jarl.md#sets-from-jarl)). housecarld reads the names' A and
+  AAAA records through Jarl's API (CNAMEs followed, one view per name)
+  and keeps the addresses that lie within the zones and networks the set
+  allows; Jarl never calls Housecarl. What was last read is kept (in the
+  database too) while Jarl can't be reached. A change of the addresses
+  applies the ruleset again without a history entry, as dynamic sets'
+  entries aren't in it either. Names Steward makes from leases must not
+  be used: a client chooses its own hostname.
+* **The web UI's certificate** comes through Jarl, as Steward's does
+  ([certificates.md](certificates.md)).
 
-## Open questions
+## Not in 0.1
 
-* IPv6 from the start, or IPv4 first as Steward did?
-* Import an existing nftables (or iptables) ruleset, as Jarl imports a BIND
-  configuration?
-* Connection tracking defaults: established and related traffic allowed
-  everywhere unless a rule says otherwise?
-* Does Housecarl also run the router's interfaces and routes, or only the
-  firewall? (Only the firewall, for now.)
-* Rate limits and flow offloading: later.
+* Interfaces, addresses and routes: Housecarl only runs the firewall.
+* iptables rulesets (only nftables can be imported).
+* Rules by MAC address, by user, or by DNS name.
+* Flow offloading to hardware (the software flowtable is there).

docs/development.md

@@ -0,0 +1,152 @@
+# Developing Housecarl
+
+## Layout
+
+```
+cmd/housecarld/       the daemon: serve, init, user, check, backup, restore, config
+cmd/hc/               the CLI
+internal/
+  model/              the configuration (State: settings, zones, sets, services, rules, NAT), users, history
+  validate/           whole-configuration checks with stable codes (docs/validation.md)
+  render/             State → the nftables ruleset text, with each line's object
+  nft/                runs nft: check, apply (keeping dynamic set entries and counters), list, the boot file
+  nflog/              reads logged packets from netlink log group 4171
+  linkwatch/          netlink: interfaces that appear or go (the flowtable's devices)
+  tablewatch/         netlink: a table deleted (shadow mode puts Housecarl's back; docs/shadow.md)
+  importer/           nft -j list ruleset → State, with notes and raw rules (docs/import.md)
+  hostif/             the machine's interfaces
+  store/              SQLite (modernc.org/sqlite, no cgo): configuration, history, users, tokens, sessions, audit
+  engine/             configuration changes: edit a copy, diff, validate, check and apply, save with history, undo
+  server/             the HTTP API and the web UI, authentication, the local socket, probation, the log ring
+  api/, client/       JSON types and the Go client shared by server and CLI
+  auth/               passwords (argon2id), tokens, sessions, grants and roles
+  certs/, jarl/       the web UI's certificate through Jarl (docs/certificates.md); jarl/ is the Jarl API client
+  jarlsets/           sets from Jarl: reads the names, follows CNAMEs, keeps what was read, within (docs/jarl.md)
+  config/             housecarld's YAML configuration
+  version/            what a build reports
+  web/static/         the web UI: plain ES modules, no build step, embedded with go:embed
+deploy/               systemd units, sysusers.d and tmpfiles.d files, Debian package scripts
+scripts/e2e.sh        end-to-end test with real traffic through a router container
+scripts/fakejarl/     the bit of Jarl's API sets from Jarl read, for the e2e test
+scripts/mkdeb.sh      the Debian packages (make deb)
+scripts/logo.py       the web UI's logo from artwork/logo.svg
+scripts/banner.py     the README's banner from the web UI's logo and wordmark
+```
+
+[design.md](design.md) describes how the pieces fit.
+
+## Building and testing
+
+```sh
+make            # bin/housecarld, bin/hc, bash completion
+make test       # go vet, gofmt check, go test ./...
+make e2e        # scripts/e2e.sh: needs Docker
+make deb        # dist/housecarl_<version>_<arch>.deb and housecarl-cli_...
+```
+
+**Never apply a test ruleset to your own machine's nftables.** Tests that
+need the real `nft` run it in a container.
+
+**Unit tests** need no nftables: the API tests (`internal/server`) use a
+fake firewall. The render tests compare the ruleset
+with what is expected, line by line, and the importer tests import the
+maintainer's router ruleset (`internal/importer/testdata`) and check the
+result.
+
+**Real nftables**: with `HOUSECARL_NFT` set to a command that runs nft
+somewhere harmless, the render, nft and importer tests also have nft
+check (and apply) their rulesets. A `debian:trixie` image with nftables
+1.1.3 (Debian 13's) is the reference:
+
+```sh
+make e2e         # makes the housecarl-nft image, if missing
+export HOUSECARL_NFT="docker run --rm -i --cap-add NET_ADMIN housecarl-nft nft"
+export HOUSECARL_NFT_IMAGE=housecarl-nft
+make test
+```
+
+Newer nftables accepts things 1.1.3 doesn't (a meter with a timeout for
+`ct count`, hour ranges past midnight): keep to what 1.1.3 takes.
+
+**The e2e test** builds a small network in Docker from the same image:
+
+* a router container (housecarld, the `hc` CLI) on three networks named
+  `wan`, `enp2s0` and `wg0` inside it, IPv4 and IPv6;
+* "the internet" on the WAN side, a LAN client, and a "jellyfin" host.
+
+It imports a copy of the maintainer's ruleset (lab addresses) with
+housecarld in shadow mode beside it (traffic unchanged, a would-be drop
+logged and let through, the table back after the old ruleset is
+reloaded), restarts it enforcing, takes over the old table, and checks
+with real traffic: port forwards with
+hairpin NAT and the IPv6 pinhole, masquerading, antispoofing, the drop
+log, a lockout change that rolls back by itself, a restart, the boot
+ruleset after a flush, and that other tables are left alone. It makes
+the `housecarl-nft` image itself if it is missing.
+
+`KEEP=1` leaves the lab running (the web UI on `10.88.10.1:8070`);
+`DOCKER_HOST` points at another Docker daemon. Root isn't needed on the
+host.
+
+The systemd sandbox (capabilities, the boot unit) is only tested by
+installing the package in a container that runs systemd
+(`--privileged --cgroupns=host /sbin/init`).
+
+## Certificates
+
+As in Steward: `internal/certs` has unit tests with a fake Jarl and an
+in-process DNS server; a real order is tested against Jarl's e2e
+environment and Pebble (Steward's docs/development.md has the steps).
+
+## The web UI
+
+Plain JavaScript modules under `internal/web/static/`, built with the
+same helpers as Jarl's and Steward's:
+
+* All DOM goes through `h()` in `ui.js`, never `innerHTML`.
+* The Content-Security-Policy allows only the app's own scripts and
+  styles, so no inline styles. Use a class in `style.css`, or set
+  `el.style.prop`.
+* The files are embedded, so rebuild housecarld after changing them.
+
+There are no JavaScript tests: bundle `assets/app.js` with esbuild to
+catch syntax and import errors, and look at the pages in a browser
+against the e2e lab.
+
+## Conventions
+
+* Every user-visible change goes into `CHANGELOG.md` under "Unreleased",
+  with the docs updated in the same change.
+* Errors say what is wrong and how to fix it, naming the object.
+* New validation rules get a stable code, a test, and a row in
+  [validation.md](validation.md).
+* Configuration changes go through the engine, never straight to the
+  store, so they are validated, checked by nftables, recorded and
+  undoable.
+* Schema changes append a migration to `internal/store/store.go`; never
+  edit a shipped one.
+* `gofmt` and `go vet` clean (`make test`).
+
+## Releasing
+
+As for Jarl and Steward:
+
+1. Move "Unreleased" in `CHANGELOG.md` under `## [<version>] - <date>`.
+   A release candidate may leave it there.
+2. Put the version in `VERSION`, and update the README's status line.
+3. Run `make test && make e2e`.
+4. Commit as "Release X", tag it `git tag -a vX -m "Housecarl X"`, and push.
+
+A fix to a release candidate may go out as a build of it rather than a
+new candidate: `0.2.0-rc.3+git<count>-<commit>`, where `<commit>` is the
+short ID of the last commit before "Release X" and `<count>` is `git
+rev-list --count` of it. The count orders the builds (apt compares it as a
+number; the ID alone would order them at random); each sorts after
+`0.2.0-rc.3` and before `0.2.0-rc.4`. Debian writes it
+`0.2.0~rc.3+git<count>~<commit>`.
+
+The tag starts `.gitea/workflows/release.yml`, which checks the tag
+against `VERSION` and the CHANGELOG, runs the tests, builds the tarballs,
+the CLI for each OS and the `.deb`s, and publishes them
+(`scripts/gitea-release.sh`). It needs the repository secret
+`RELEASE_TOKEN` (`write:repository`, `write:package`).

docs/flowtable.md

@@ -0,0 +1,187 @@
+# The flowtable: a fast path for forwarded traffic
+
+The flowtable is a shortcut inside the kernel for traffic the firewall
+**forwards** (between the internet and your LAN, between zones). It
+speeds up routing on a machine whose CPU can't keep up with its line.
+It doesn't make the firewall any safer or less safe, but it changes what
+some features can see, so read [when not to use it](#when-not-to-use-it)
+before you turn it on.
+
+It is off until you name its devices.
+
+## What it does
+
+Without a flowtable, every forwarded packet takes the whole way through
+the kernel: the prerouting hooks, the routing decision, Housecarl's
+forward chain (the built-ins, then your rules) and postrouting with NAT.
+For an established connection almost all of that work only ends in the
+`established` built-in accepting it again, packet after packet.
+
+With a flowtable, Housecarl adds one built-in to the forward chain:
+
+```
+meta l4proto { tcp, udp } flow add @ft
+```
+
+A TCP or UDP connection takes the whole way for its first packets, as
+always: your rules decide whether it is allowed at all. Once the kernel
+has seen it established (and your rules let it through), it is added
+to the flowtable. From then on its packets are caught as soon as they
+arrive on one of the flowtable's devices (the *ingress* hook) and sent
+straight out on the other side: NAT is still applied, the TTL still
+counts down, but every chain in between is skipped.
+
+```
+                 first packets                         every packet after
+in ─► prerouting ─► routing ─► forward ─► postrouting ─► out
+ │                                                        ▲
+ └────────────── flowtable (ingress) ─────────────────────┘
+```
+
+It is the software flowtable, run by the CPU. Offloading to the network
+card's hardware isn't supported ([design.md](design.md#not-in-01)).
+
+## When to use it
+
+Turn it on when **the router's CPU is what limits your speed**:
+
+* a speed test through the router gets less than your line gives (or
+  less than the same test from the router itself), and
+* during it, one CPU core is busy with network work: in `top`, press `1`
+  for every core, and look at `si` (softirq) near 100 % on a core.
+
+Typical cases: a small or old machine (Atom, Celeron, a Raspberry Pi, an
+old Core 2) on a fast line (500 Mbit/s and up), lots of small packets
+(gaming, VoIP, many downloads at once), or a machine that is busy with
+other work besides routing. The gain is CPU time per packet: often well
+over half of the forwarding cost, which shows up as speed only when the
+CPU was the limit, and otherwise as a cooler, quieter, more responsive
+machine.
+
+In shadow mode ([shadow.md](shadow.md)) Housecarl leaves its flowtable
+out: an offloaded connection skips every table's forward chain, the
+other firewall's too. If that firewall has a flowtable of its own, the
+same goes for Housecarl's forward chain: it only sees each connection's
+first packets.
+
+## When not to use it
+
+**When the CPU keeps up anyway.** If the speed test already reaches your
+line and `si` stays low, the flowtable buys you nothing you'd notice,
+and costs you the figures below. Most modern x86 machines route a
+gigabit line without it.
+
+**When you count or limit traffic in the forward chain.** The fast path
+skips the forward chain, so whatever must see *every* packet of a
+connection only sees the first few. Housecarl warns about each:
+
+| Feature | With the flowtable | Warning |
+|---|---|---|
+| quotas and byte-rate limits on forward rules | only count first packets: a device can go far past its allowance | `flowtable_traffic` |
+| forward rules with "every packet" (`--every-packet`) | only see first packets | `flowtable_traffic` |
+| statistics' traffic per device (`stats_per_device`) | only count first packets | `stats_flowtable` |
+| hit counters of the `established` built-in (forward) | stop rising once a connection is offloaded | |
+
+Rate limits by packets (`--limit 10/second`) and connection limits act
+on new connections, so they work the same.
+
+**When other tables must see forwarded traffic.** Offloaded packets skip
+the forward hooks of *every* table, not only Housecarl's: Docker's,
+Incus', libvirt's or a table of your own. Those normally only judge new
+connections, which still pass through them, but a table that counts or
+shapes established traffic in its forward chain won't see it.
+
+What keeps working as before:
+
+* **Your rules** decide every new connection: nothing gets offloaded
+  that your rules didn't let through. A rule you add later doesn't stop
+  a connection already open, but that is true without the flowtable too
+  (the `established` built-in accepts it).
+* **NAT**: port forwards, hairpin NAT, masquerading and SNAT.
+* **The input and output chains**: traffic to and from the firewall
+  itself never uses the flowtable.
+* **Closing connections** (`hc connections close`, Traffic →
+  Connections) ends their fast path too: the kernel drops a flow with
+  its tracking entry.
+* **Interface statistics** (traffic per zone and interface) come from the
+  interfaces' own counters, which count every packet.
+* **Traffic shaping with `tc`** on the outgoing interface still applies.
+* **The log** and **the packet tracer** see what goes through the rules:
+  new connections, as always. The tracer doesn't see the packets of an
+  offloaded connection; trace a new one.
+* ICMP, and protocols other than TCP and UDP (GRE, ESP, SCTP), always
+  take the normal way.
+
+## How to turn it on
+
+Name the devices whose traffic should take the fast path, **both
+sides**: the interface connections come in on and the one their replies
+come in on. For a router that means the external interface and the
+inside ones:
+
+```sh
+hc settings set --flowtable wan --flowtable enp2s0
+hc settings                     # Flowtable: wan, enp2s0
+```
+
+In the web UI: Settings → Flowtable, add each device. The fast path
+only catches packets arriving on a listed device, so a connection to or
+from an unlisted one gains little or nothing: leave a device out only on
+purpose (say, a WireGuard interface whose traffic you'd rather keep on
+the normal way).
+
+* Use **exact device names**, not wildcards (`incusbr*` is refused):
+  the kernel attaches the flowtable to each device by name.
+* With **PPPoE**, name the PPP interface (`ppp0`, or the name your
+  `ppp@` unit gives it), the one in your external zone.
+* `--flowtable` replaces the whole list; `hc settings set --flowtable
+  none` turns the flowtable off.
+
+The change is applied (and on probation, if made over the network) like
+any other.
+
+### Devices that come and go
+
+PPP and WireGuard interfaces don't exist until their service brings
+them up, and PPPoE makes a new `ppp0` each time it reconnects. Housecarl
+handles that:
+
+* A device that doesn't exist yet is left out of the flowtable, with the
+  warning `flowtable_device_missing`, instead of failing the whole
+  ruleset.
+* housecarld watches the kernel's interface changes: when a flowtable
+  device appears, or comes back as a new interface after a reconnect,
+  it applies the ruleset again with it, a couple of seconds later. The
+  journal says `interfaces changed: applied the ruleset again`.
+  Applying again keeps the counters, dynamic sets and quotas, as any
+  change does, and adds no history entry.
+* The ruleset saved for boot has no flowtable at all: it loads before
+  the network is up, when `ppp0` and `wg0` can't exist yet, and
+  nftables on Debian 13's kernel refuses a flowtable naming a device
+  that doesn't exist (and with it the whole file). housecarld adds the
+  flowtable once it starts. Until then, traffic takes the normal way,
+  through the same rules.
+
+## Checking it works
+
+```sh
+sudo nft list flowtables          # the devices it is attached to
+sudo conntrack -L | grep OFFLOAD  # connections on the fast path
+```
+
+`conntrack` is in Debian's `conntrack` package. A long download through
+the router should show up as `[OFFLOAD]`. Compare a speed test, and `si`
+in `top`, before and after.
+
+If nothing gets offloaded, check that both of the connection's devices
+are listed, that `flowtable_device_missing` isn't in `hc status`, and
+that the connection is TCP or UDP and established.
+
+## Turning it off
+
+```sh
+hc settings set --flowtable none
+```
+
+Connections on the fast path go back to the normal way with their next
+packet.

docs/import.md

@@ -0,0 +1,213 @@
+# Importing an nftables ruleset
+
+`hc import` turns the ruleset you have into Housecarl's configuration,
+and `hc takeover` deletes the old table once Housecarl's replaces it.
+Until then both apply.
+
+## 1. Look first
+
+On the firewall, as root:
+
+```sh
+sudo hc import --defines /etc/nftables.conf -n
+```
+
+`hc` runs `nft -j list ruleset` (and `nft -a list ruleset`, for the text
+of rules it can't translate) and sends both to housecarld. `-n` (dry run)
+shows what it would make, without saving anything: zones, sets,
+services, rules, port forwards and NAT, then **notes** on every guess it
+made, and the rules it **kept raw**.
+
+`--defines` reads your ruleset file's `define` lines, so sets and zones
+get your names ([How things are named](#how-things-are-named)).
+`--zone enp2s0=lan` names a zone yourself.
+
+From another machine, give the JSON as a file, and the text with
+`--text`:
+
+```sh
+ssh fw 'sudo nft -j list ruleset' > ruleset.json
+ssh fw 'sudo nft -a list ruleset' > ruleset.txt
+hc import ruleset.json --text ruleset.txt --defines nftables.conf -n
+```
+
+With several tables, `--table "inet main"` picks one; by default every
+table but Housecarl's is read. The web UI has the same under Server →
+Import a ruleset.
+
+## How things are named
+
+**Zones** are named in this order:
+
+1. **As you say**: `--zone enp2s0=lan` (repeat it; Zone names on the
+   web UI's Import page). Interfaces given the same name share one zone
+   (`--zone enp2s0=lan --zone enp3s0=lan`).
+2. **After a define holding the interface**, with a trailing `_int`,
+   `_if`, `_iface`, `_ifname`, `_interface`, `_dev`, `_device` or `_nic`
+   taken off: `define wan_int = "ppp0"` makes the zone `wan`, `define
+   iot_if = enp2s0.20` (a VLAN) the zone `iot`.
+3. **After the interface itself**: `enp2s0`.
+
+If the name is taken, the zone is called `INTERFACE_zone`. A define
+names what it was *for* in your old ruleset, which isn't always what the
+network is (`define dhcp_int = enp2s0` makes `dhcp`, for your LAN). A
+note lists each zone and where its name came from; rename one with
+`--zone` and import again, or afterwards with `hc zone update dhcp
+--name lan` (its rules follow). Name zones for what the network is for:
+`lan`, `iot`, `guest`, `servers`, `vpn`, `wan`.
+
+**Address sets**: a named set of the table keeps its name; a define
+holding a list of addresses, or one prefix, gives its name to a set
+(`define trusted = { 10.0.10.0/24, 10.1.1.0/24 }` makes `trusted`,
+wherever those addresses appear together). A single address stays an
+address. When two defines hold the same addresses, the first by name
+wins.
+
+**Services**: a define holding several ports names a service (`define
+web_ports = { 80, 443 }` makes `web_ports`); one port stays a port
+(`tcp/22`).
+
+**Port forwards** are named after their comment, without a leading
+"Port forward" (`comment "Port Forward Jellyfin"` makes `jellyfin`);
+several ports in one DNAT become one forward each (`web-80`, `web-443`),
+and a DNAT map's elements are named after the map (`forwards-443`).
+
+Names that aren't valid are made so: other characters become `_` (`-`
+in forward names), and a name must start with a letter.
+
+## What becomes what
+
+* **Interfaces** become zones. Zones are marked external when the rules
+  show it: MSS clamping on the way out, DHCPv6 client replies, ping
+  accepted from everywhere else.
+* **Sets and defines** of addresses become address sets; port lists
+  become services.
+* **Jumps and gotos** to your own chains are flattened: the jump's
+  condition is added to each rule of the chain, in place. When the jump
+  and a rule both match sources (or destinations), the narrower is kept
+  if one lies within the other; otherwise, and for any rule there that
+  stays raw, the jump's condition is written in front of the raw rule's
+  text, so it still only applies where the jump did.
+* **Rules** keep their order. A rule that is a built-in (loopback,
+  established, invalid, the ICMPv6 rules, MSS clamping, the DHCPv6 client
+  rule, antispoofing) becomes that built-in only when it sits where the
+  built-in goes, at the head of every chain it covers. Otherwise it stays
+  a rule where it was, so the ruleset does what it did. Since built-ins
+  come before every rule, once one at a chain's head stays a rule, the
+  ones after it there stay rules too, in order: an invalid drop that
+  stays a rule keeps the ICMPv6 accepts after it behind it, as they were.
+  Built-ins your ruleset doesn't have are turned off.
+* **DNAT** rules become port forwards, TCP and UDP of the same port
+  merged into one, one per port when a rule has several. The
+  forward-chain accepts that let them through (`ct status dnat`) become
+  part of the port forward; an accept to the same host and port without
+  `ct status dnat` lets in more than forwarded connections, and stays a
+  rule. A masquerade for inside sources to the same host becomes its
+  hairpin. An IPv6 accept to the same port, from everywhere, becomes the
+  forward's IPv6 pinhole when there is only one candidate. A DNAT to an
+  IPv6 address, or with matches a port forward has no field for, is left
+  out with a note.
+* **Maps are expanded**, one element at a time. A DNAT map
+  (`dnat ip to tcp dport map @forwards`, named or written in the rule)
+  becomes one port forward per element; forwards from a named map are
+  called after it (`forwards-443`). A verdict map
+  (`tcp dport vmap { 22 : accept, 23 : drop }`) becomes one rule per
+  element, in place, with `jump`s inlined as above. A lookup matches at
+  most one element, so the rules do the same. A map keyed by several
+  fields (`ip saddr . tcp dport`) can't be expanded: a rule using a named
+  one is left out with a note, to be added by hand.
+* **Masquerade and SNAT** become source NAT; **redirect** becomes a
+  redirect.
+* **Meters** (`update @m { ip saddr limit rate over 3/minute }`,
+  `add @m { ip saddr ct count over 20 }`, quotas in sets) become the
+  rule's per-source (or per-destination) limit, connection limit or quota;
+  a mask in the key (`ip saddr and 255.255.255.0`) becomes its grouping.
+  Byte rates and quotas are imported too; a quota imported resets only by
+  hand until you give it a period. A rule with a quota or a byte rate that
+  accepts stays a raw rule ([rules.md](rules.md#bandwidth-and-quotas)).
+* **Sets with `flags timeout`** (filled by a program or a rule) become
+  dynamic sets, without their entries of the moment: whatever fills them
+  must fill Housecarl's from then on, as a note says.
+* **Policies** become the chains' policies.
+* **Anything else** becomes a raw rule with its original text, in place:
+  a match Housecarl has no field for, two matches on the same field
+  (`ip saddr 10.0.0.0/8 ip saddr != 10.0.10.0/24`), `!=` where Housecarl
+  only has `==`, a set of interface names. Its set references are made
+  to work in Housecarl's table: an imported address set becomes
+  `@NAME_v4` or `@NAME_v6`, any other set (interface names, ports) is
+  written out with its elements. A raw rule naming a set that can't be
+  (a meter, an empty set) is left out with a note.
+* **Logging rules** log to Housecarl's log (`hc log`), not the kernel's.
+
+Several base chains on one hook (`input` at priority -10 and at 0) are
+put one after the other, by priority, into Housecarl's one chain, with
+the stricter policy; a note says to check it, since a packet had to get
+past each, and an accept in the first no longer ends there. A prerouting
+filter chain after connection tracking becomes the early chain, which
+runs before it: a note says so. Chains on hooks Housecarl doesn't manage
+(ingress, a bridge table) are left out, with a note.
+
+## 2. Import
+
+```sh
+sudo hc import --defines /etc/nftables.conf
+sudo hc ruleset                  # what Housecarl now applies
+sudo hc status                   # problems, and the other tables
+```
+
+Housecarl's table is now in force **alongside** the old one: a packet
+must be accepted by both. Nothing should change for your traffic. If
+Housecarl already has a configuration, `--replace` replaces it.
+
+To be sure first, import in **shadow mode** ([shadow.md](shadow.md)):
+Housecarl's ruleset is then loaded but drops nothing, and its log shows
+each packet the old ruleset lets through that Housecarl would drop. When
+there are none you mind, leave shadow mode and take over.
+
+Imported over the network (not through the local socket), the import is
+on probation like any change: confirm it.
+
+## 3. Take over
+
+Once the rules look right (and their hits in the web UI agree), delete
+the old table:
+
+```sh
+sudo hc takeover "inet main"
+```
+
+From now on only Housecarl's rules apply. Until you do, the dashboard
+and `hc status` warn that the imported table still applies
+(`takeover_pending`); they also warn while `nftables.service` is enabled
+(`nftables_service`). Over the network, this is on
+probation too: unless confirmed, the old table comes back.
+
+## 4. The boot
+
+Housecarl's ruleset is loaded at boot by `housecarl-ruleset.service`.
+Debian's `nftables.service` loads `/etc/nftables.conf`, which usually
+starts with `flush ruleset`: that would delete Housecarl's table, or
+bring the old one back. Retire it:
+
+```sh
+sudo systemctl disable nftables.service
+sudo mv /etc/nftables.conf /etc/nftables.conf.before-housecarl
+```
+
+Disable it, but **don't stop it** (no `systemctl stop nftables`, no
+`disable --now`): Debian's unit stops by running `nft flush ruleset`,
+which deletes every table, Housecarl's and Docker's included, at once.
+Disabled, it simply doesn't run at the next boot.
+
+`housecarl-ruleset.service` runs after `nftables.service` in any case,
+so a leftover `flush ruleset` can't undo it. Docker, Incus and libvirt
+keep adding their own tables as before.
+
+## Checking the result
+
+* `hc rule list input` (and `forward`) side by side with
+  `nft -a list table inet main`: the same rules, in the same order, with
+  built-ins above.
+* The notes: each says what to check.
+* `hc counters` after a while: a rule with no hits that had hits before
+  is worth a look.

docs/install.md

@@ -0,0 +1,142 @@
+# Installing Housecarl
+
+Housecarl runs on Linux with nftables 1.0 or newer. It is made for and
+tested on Debian 13 (nftables 1.1.3). The CLI also runs on macOS and
+Windows.
+
+## From the package registry
+
+The release workflow publishes Debian packages to the Gitea registry,
+`stable` for releases and `testing` for release candidates. Jarl's and
+Steward's packages come from the same registry. If apt already has it,
+`sudo apt install housecarl` is all it takes. Otherwise add it:
+
+```sh
+sudo install -d /etc/apt/keyrings
+sudo curl -fsSL https://gitea.adbraun.lol/api/packages/adam/debian/repository.key -o /etc/apt/keyrings/gitea-adam.asc
+echo "deb [signed-by=/etc/apt/keyrings/gitea-adam.asc] https://gitea.adbraun.lol/api/packages/adam/debian stable main" \
+  | sudo tee /etc/apt/sources.list.d/gitea-adam.list
+sudo apt update
+sudo apt install housecarl
+```
+
+For release candidates too, add the `testing` distribution: the same
+`deb` line with `testing` instead of `stable`.
+
+There are two packages:
+
+* **`housecarl`** is the server: `/usr/bin/housecarld`, the systemd units
+  `housecarl.service`, `housecarl-ruleset.service` (the ruleset at boot),
+  `housecarl-backup.service` and `housecarl-backup.timer`, and the
+  `housecarl` user and directories. It creates
+  `/etc/housecarl/housecarl.yaml` on the first install and never touches
+  it again. It depends on `housecarl-cli` and `nftables`.
+* **`housecarl-cli`** is the CLI, `/usr/bin/hc`. It can be installed
+  alone on any machine to manage Housecarl remotely.
+
+Installing changes nothing in the firewall: housecarld applies a ruleset
+only once it has a configuration.
+
+The release page also has `.deb` files, tarballs and the CLI for Linux,
+macOS and Windows.
+
+## From source
+
+```sh
+make                        # bin/housecarld, bin/hc (Go, as in go.mod)
+sudo make install           # /usr/local/bin, units in /etc/systemd/system
+sudo systemd-sysusers housecarl.conf && sudo systemd-tmpfiles --create housecarl.conf
+sudo chgrp housecarl /etc/housecarl /etc/housecarl/housecarl.yaml && sudo systemctl daemon-reload
+```
+
+`make deb` builds the packages (`DEB_ARCH=arm64` for another architecture).
+
+## First steps
+
+Do this from a console, or over SSH with a way back in if it goes wrong:
+you are changing the firewall you are connected through.
+
+1. **Settings** are in `/etc/housecarl/housecarl.yaml`
+   ([configuration.md](configuration.md)). By default the web UI and API
+   listen on `127.0.0.1:8070`. To reach them from your network, add an
+   address there, ideally with TLS or behind a TLS reverse proxy.
+2. **Create the database and the first administrator:**
+
+   ```sh
+   sudo -u housecarl housecarld init
+   ```
+
+   It asks for the administrator's name (`admin` if you just press Enter;
+   your own name is better) and password.
+3. **Start Housecarl:**
+
+   ```sh
+   sudo systemctl enable --now housecarl
+   ```
+
+   Until it has a configuration, housecarld applies nothing: the machine
+   stays as open (or as closed) as it was.
+4. **Import your ruleset**, if you have one ([import.md](import.md)):
+
+   ```sh
+   sudo hc import --defines /etc/nftables.conf -n      # look
+   sudo hc import --defines /etc/nftables.conf         # import
+   sudo hc takeover "inet main"
+   sudo systemctl disable nftables.service
+   ```
+
+   To try Housecarl beside your firewall first, turn on **shadow mode**
+   before the import: it loads Housecarl's ruleset but blocks nothing,
+   and shows what it would do ([shadow.md](shadow.md)).
+
+   **Or start from scratch**, through the local socket (no probation).
+   The first change puts the ruleset in force, and the input and forward
+   policies drop what no rule accepts, so let yourself in first:
+
+   ```sh
+   sudo hc set add admins 192.168.1.0/24
+   sudo hc rule add input --from admins --service ssh
+   sudo hc rule add input --from admins --service tcp/8070 --comment "Housecarl"
+   sudo hc zone add wan --interface ppp0 --external
+   sudo hc zone add lan --interface eth1
+   sudo hc rule add input --in lan --service dns --service dhcp
+   sudo hc rule add forward --in lan --out wan
+   sudo hc snat add --out wan
+   sudo hc settings set --builtin mss_clamp=on --own-prefix 2001:db8::/48
+   sudo hc status
+   ```
+
+   Housecarl doesn't add a rule that keeps SSH or the web UI open: you
+   write it. Connections already open (your SSH session) carry on, since
+   established connections are accepted.
+5. **Open the web UI** on an address in `listen`, and log in.
+
+## From another machine
+
+Install `housecarl-cli`, make a token, and set the server:
+
+```sh
+sudo hc token create laptop               # on the firewall; prints the token once
+hc config set server https://fw.example.com:8070
+hc config set token                       # paste it
+hc status
+```
+
+Changes from there are on probation: `hc confirm` keeps them.
+
+## Upgrading
+
+`apt upgrade` restarts a running housecarld, which applies the ruleset
+again (counters keep counting) and upgrades its database when it starts.
+The firewall stays up during the upgrade: the ruleset belongs to the
+kernel, not to housecarld. Take a backup first if you like
+(`sudo -u housecarl housecarld backup`); the daily timer keeps 30 days of
+them.
+
+## Removing
+
+`apt remove housecarl` stops housecarld and disables the boot unit. The
+ruleset stays in force until the next boot, or until you delete it:
+`sudo nft delete table inet housecarl`. A purge keeps the database and
+the backups; remove `/var/lib/housecarl` and `/var/backups/housecarl` by
+hand if you no longer need them.

docs/jarl.md

@@ -0,0 +1,297 @@
+# Connecting Housecarl to Jarl
+
+Housecarl uses [Jarl](https://gitea.adbraun.lol/adam/jarl), the BIND
+manager, for two things:
+
+* the DNS-01 challenge for the web UI's certificate
+  ([certificates.md](certificates.md)): Housecarl writes a TXT record;
+* **sets from Jarl** ([below](#sets-from-jarl)): sets whose addresses are
+  those of names in Jarl, such as `nas.adbraun.lol`, kept up to date.
+  Housecarl only reads.
+
+Housecarl talks to Jarl's REST API with an API token. Give it a **user of its
+own**, with **only the permission it needs**. Jarl never calls Housecarl.
+
+## 1. A user for Housecarl, in Jarl
+
+On the Jarl server, make a user without a password. It can't log in to
+Jarl's web UI, only use its tokens.
+
+For **sets from Jarl**, grant it reading the records of the zones and views
+your names are in:
+
+```sh
+jarl user create housecarl --no-password --comment "Housecarl on fw01" \
+    --grant "records:read zones=adbraun.lol views=internal"
+```
+
+Include every zone a name's CNAMEs lead to as well (`zones=adbraun.lol,wg.adbraun.lol`).
+Reading is all it needs: it can't change anything in Jarl.
+
+For the **certificate**, grant it the challenge record of Housecarl's own
+name, and nothing else:
+
+```sh
+jarl user create housecarl --no-password --comment "Housecarl on fw01" \
+    --grant "records:write zones=adbraun.lol names=_acme-challenge.fw.adbraun.lol types=TXT"
+```
+
+For both, give both grants (`--grant` twice). `jarl user update housecarl
+--grant ... --grant ...` replaces a user's grants with the ones given.
+
+Use your zone for `adbraun.lol`, and the name Housecarl's web UI is reached
+by for `fw.adbraun.lol` (`tls.acme.domain`). The grant lets the token add
+and remove TXT records at exactly `_acme-challenge.fw.adbraun.lol`:
+
+* nothing else in the zone (`names=`);
+* no other record type (`types=TXT`);
+* no other zone (`zones=`).
+
+It can see that the zone exists, which it needs to find it, but it can't
+read the zone's other records.
+
+Why not something wider:
+
+* **Jarl's `acme` role** (`records:write names=_acme-challenge.* types=TXT`)
+  would let the token publish challenges for *any* name in *any* zone, and
+  so obtain certificates for all of them. Housecarl needs only its own name.
+* **Your own account's token** would carry all your permissions, and
+  Housecarl's certificate would stop renewing the day your account changes.
+  A user per program keeps the two apart, and the audit log shows
+  `housecarl/...` for what Housecarl did.
+
+If the challenge name is a CNAME into a zone kept for challenges, such as
+`_acme-challenge.fw.adbraun.lol CNAME fw.acme.adbraun.lol.`, grant the
+target instead: `zones=acme.adbraun.lol names=fw.acme.adbraun.lol`.
+
+## 2. The token
+
+Make the token, and copy it to the Housecarl server into a file only the
+`housecarl` user can read:
+
+```sh
+jarl token create fw01 --user housecarl -q > housecarl.token          # on the Jarl server, or with the jarl CLI anywhere
+scp housecarl.token fw01:
+ssh fw01 'sudo install -m 0600 -o housecarl -g housecarl housecarl.token /etc/housecarl/jarl.token && rm housecarl.token'
+rm housecarl.token
+```
+
+A token file is better than putting the token in `housecarl.yaml`: the
+configuration file is readable by the `housecarl` group, and gets copied
+around more readily.
+
+You can give the token an expiry (`--expires 365d`). Rotate it in Jarl with
+`jarl token rotate ID --grace 1d`, then put the new secret in the file and
+restart housecarld within the grace period. Since the certificate is renewed
+every couple of months, nothing breaks while you do. Sets from Jarl keep
+their addresses while the token doesn't work (they warn, see
+[below](#when-jarl-cant-be-reached)), so they don't break either.
+
+## 3. Housecarl's settings
+
+In `/etc/housecarl/housecarl.yaml`:
+
+```yaml
+jarl:
+  server: https://dns.adbraun.lol:8053
+  token_file: /etc/housecarl/jarl.token
+  # ca_cert: /etc/housecarl/jarl-ca.pem   # if Jarl's certificate isn't from a public CA
+  # refresh: 1m                           # how often sets from Jarl read their names
+```
+
+Then restart housecarld (`systemctl restart housecarl`). Housecarl needs to
+reach Jarl's API over the network: Jarl's `listen` must include an
+address Housecarl can reach, ideally over HTTPS.
+
+`hc cert check` ([certificates.md](certificates.md#checking)) tests
+the connection, the token's permission, and that the record reaches the
+internet.
+
+## Sets from Jarl
+
+A set can take its addresses from Jarl instead of listing them: you name
+DNS names, and the set holds their addresses. Rules use it like any other
+set. When a record changes in Jarl, the set follows within a minute,
+without a change to Housecarl's configuration.
+
+```sh
+hc set add nas --jarl nas.adbraun.lol@internal --within lan
+hc rule add forward --in wg --to nas --service smb
+hc set jarl nas                       # what it holds, name by name
+```
+
+In the web UI: Sets → Add set, kind "from Jarl". The **Addresses…**
+button shows what the set holds.
+
+A set from Jarl has:
+
+* **names**: one or more names, each `NAME@VIEW`, or `NAME` when only one
+  of Jarl's views has it ([Views](#views));
+* **within**: where its addresses may be, one or more zones or networks
+  ([Within](#within)). It is required.
+
+It has no entries of its own, and isn't dynamic. It can be renamed (it goes
+by its names in Jarl, not by its own).
+
+### What a name gives
+
+Housecarl reads the name's records as named would answer them in the
+view:
+
+* **Every A and AAAA record** of the name. A name with three addresses
+  puts all three in the set, IPv4 and IPv6 alike; a rule using the set
+  matches each family with its own addresses.
+* **CNAMEs are followed**: `files.adbraun.lol CNAME nas.adbraun.lol.` gives
+  nas's addresses. Chains are followed up to 8 CNAMEs, from zone to zone
+  (a sub-zone such as `wg.adbraun.lol` too), but always in the **same
+  view**, and only into zones Jarl holds. `hc set jarl` shows the chain:
+  `files.adbraun.lol → nas.adbraun.lol`.
+* **Disabled records** are left out, as named leaves them out.
+
+A name gives **no addresses**, and a warning (`jarl_name`) says why, when:
+
+* it has no A, AAAA or CNAME record (only an MX, say, or nothing);
+* only a **wildcard** (`*.adbraun.lol`) would answer for it: wildcards
+  aren't followed, so name a record of its own;
+* a **CNAME leads out of Jarl** (`host.duckdns.org`), into a zone Jarl
+  doesn't hold, or to a name with no address;
+* the CNAMEs **loop**, or are more than 8;
+* it is **below a delegation**: NS records in the zone hand that part to
+  other servers, so Jarl doesn't hold its records;
+* its zone is a **secondary, mirror, stub or forward zone** (Jarl doesn't
+  hold their records), or **disabled** in Jarl;
+* no zone the token can see holds it.
+
+A **reverse zone** isn't involved: Housecarl reads the A and AAAA records
+only, never PTRs.
+
+An **in-view zone** (one view serving another's zone) is followed to the
+zone it serves: `srv.adbraun.lol@vps` reads the internal view's records if
+vps's `adbraun.lol` is an in-view of internal's.
+
+### Views
+
+The same name can have different addresses in different views: Jarl's
+`internal` view may give `nas.adbraun.lol` its private address, `external`
+a public one. A set reads **one view per name**, written `NAME@VIEW`:
+
+* **Name the view your devices see.** The firewall matches the addresses
+  packets carry, and for devices inside that is the internal view's.
+* **Without a view**, the name is read from the only view that has a zone
+  for it. If zones in **more than one view** hold it, the name gives
+  nothing until you name the view (`jarl_name`: "is in Jarl's views
+  external, internal: name the view"). This counts every view whose zones
+  cover the name, even one where the name has no record: a record added
+  there later would otherwise change which view Housecarl reads.
+* **Linked zones** make no difference: a shared record has the same
+  address in every view, and a view's own records are why the view is
+  named.
+* **Without views** in Jarl, write the name alone.
+* A set can mix views: `--jarl nas.adbraun.lol@internal --jarl
+  host.adbraun.lol@vps`.
+
+A wrong view can't widen the set: an address from it that isn't
+[within](#within) is left out.
+
+### Within
+
+`--within` says where the set's addresses may be. Each entry is:
+
+* **a zone**: the networks connected to its interfaces (the addresses on
+  them, as `hc status` lists them), IPv4 and IPv6;
+* **a network, address or range**: `10.10.0.0/24`, `2001:db8:10::/64`,
+  `192.0.2.7`.
+
+An address from Jarl that isn't within any of them is **left out** of the
+set, and a warning names it (`jarl_outside`: "nas.adbraun.lol has address
+192.0.2.9, which is left out: not within lan"). The set's other addresses
+stay. So a record changed in Jarl, by mistake or by someone who shouldn't,
+can only point the set at addresses you allowed.
+
+Zones grow by themselves: when you add a VLAN to the `lan` zone, or give
+an interface another network, its addresses count at once. A new zone
+counts only once you add it to `within`. If a zone's interfaces have no
+address for a while (at boot, a link down), the networks it had last count.
+
+Some addresses never count, whatever `within` says: unspecified (`0.0.0.0`,
+`::`), loopback and multicast ones.
+
+`within 0.0.0.0/0` or `::/0` lets any address in, and gives a warning
+(`set_jarl_anywhere`): then whoever can change the names in Jarl decides
+what the set holds. Name a zone or a network instead. For an address
+outside your networks, such as a VPS's public address, name that address
+or its network.
+
+### When the addresses change
+
+Housecarl reads the names every `jarl.refresh` (a minute), right after a
+change to Housecarl's configuration (a new set or name), when an interface
+changes, and when you ask: `hc set jarl NAME --refresh`, or **Read Jarl
+now** in the web UI.
+
+When a set's addresses change, housecarld puts the ruleset in force again
+with the new ones. That isn't a change of the configuration: there is no
+history entry and no probation. It is recorded in the audit log
+(`set.jarl`, "added 10.10.0.22; removed 10.10.0.20") and in the journal.
+
+A new set holds nothing for the moment until its names are read, usually
+less than a second.
+
+**Open connections aren't closed** when an address leaves a set, as with
+any change to a set: the connection's packets are let through as part of an
+established connection (or by the flowtable). Close them with `hc
+connections close` if they must stop at once.
+
+### When Jarl can't be reached
+
+A set **keeps the addresses it last had** while Jarl can't be read: Jarl is
+down, the network to it is, or the token stopped working. Nothing is
+emptied, so rules keep working. A warning says so (`jarl_unreachable`), on
+the status page and in `hc status`, and `hc set jarl` shows when Jarl last
+answered.
+
+What was last read is saved in the database. After a restart or a reboot,
+the sets get those addresses at once, before Jarl is read again, and the
+boot ruleset ([operations.md](operations.md)) has them too.
+
+When Jarl **answers** that a record is gone, its addresses go. Only a
+failure to read keeps them.
+
+Without `jarl.server` in `housecarl.yaml`, sets from Jarl keep what they
+had (nothing if Jarl was never read) and warn (`jarl_not_configured`).
+
+### Who decides what the set holds
+
+A set from Jarl lets **whoever can change those records in Jarl** decide
+which addresses a rule matches. Keep that in mind for rules that accept:
+
+* `within` limits it to networks you chose;
+* Housecarl's token only reads ([step 1](#1-a-user-for-housecarl-in-jarl));
+* give the records' zones only to people you'd let change the firewall,
+  and check Jarl's audit log for who changed what.
+
+**Don't use names that clients choose for themselves.** A name a DHCP
+server makes from the client's own hostname belongs to whoever asks for
+it: any guest could call itself `nas`. Name records that you made by hand,
+or that come from a DHCP reservation.
+
+### Seeing what a set holds
+
+```
+$ hc set jarl nas
+Set:        nas
+Within:     lan
+Holds:      10.10.0.20, 10.10.0.21
+Jarl read:  2026-10-23 14:02:11
+
+NAME                                 VIEW      ADDRESS          USED
+files.adbraun.lol → nas.adbraun.lol  internal  10.10.0.20       yes
+files.adbraun.lol → nas.adbraun.lol  internal  10.10.0.21       yes
+files.adbraun.lol → nas.adbraun.lol  internal  2001:db8:10::20  no: not within lan
+```
+
+The web UI's **Addresses…** button on the Sets page shows the same, and the
+status page lists the warnings: names that give nothing (`jarl_name`),
+addresses left out (`jarl_outside`), Jarl not read (`jarl_unreachable`).
+The API is `GET /sets/{name}/jarl` ([api.md](api.md#sets-from-jarl)).
+

docs/nat.md

@@ -0,0 +1,86 @@
+# NAT
+
+## Port forwards
+
+A port forward publishes an inside host's service: connections to a
+public port are sent to the host, and let through the forward chain.
+One object does what takes three rules by hand (the DNAT, the forward
+accept, and the hairpin masquerade).
+
+In shadow mode ([shadow.md](shadow.md)) NAT rules translate nothing:
+they count the connections they would translate, and the other
+firewall's NAT does the work.
+
+```sh
+hc forward add 8443 --to 192.168.1.10:443 --public 203.0.113.10 --hairpin --name web
+hc forward add 27015-27030 --proto tcp,udp --to 192.168.1.20 --public 203.0.113.10 --hairpin --name game
+hc forward add 25 --to 192.168.1.25 --from mail-relays --name smtp
+hc forward list
+```
+
+| | |
+|---|---|
+| `PORTS` | the public port or range |
+| `--proto` | `tcp`, `udp` or both (`tcp,udp`); `tcp` by default |
+| `--to HOST[:PORT]` | the inside IPv4 host, and its port if it differs. A range keeps its ports |
+| `--public` | the public IPv4 addresses (or a set) to match. Without it, any address arriving on an external zone |
+| `--from` | only these sources may connect |
+| `--hairpin` | the public address also works from inside (below); `--hairpin-from` limits that to some inside sources |
+| `--to6` | an inside IPv6 host to open the port to (below) |
+
+In the ruleset, a forward is a `dnat ip to` rule in the `dstnat` chain,
+an accept in `forward` for connections it translated (`ct status dnat`),
+placed after the built-ins and before your forward rules, so a later
+"drop everything from wan" rule doesn't stop it.
+
+### Hairpin NAT
+
+From inside, a client that connects to the public address (because DNS
+gives it) has its connection sent to the inside host, which would answer
+it directly, and the client would drop the answer. With `--hairpin`,
+Housecarl also masquerades those connections, so the answers come back
+through the firewall. It needs `--public`: without the public address,
+there is nothing to recognise those connections by.
+
+### IPv6
+
+IPv6 needs no translation: inside hosts have public addresses. `--to6`
+opens the same port (or the `--to` port, if given) to an IPv6 host:
+
+```sh
+hc forward add 8443 --to 192.168.1.10:443 --to6 2001:db8:1::10 --public 203.0.113.10 --name web
+```
+
+IPv6 clients connect to `[2001:db8:1::10]:443` directly. A forward
+can have only `--to6`, to open an IPv6 port alone.
+
+## Source NAT
+
+Source NAT rewrites the source of traffic leaving the firewall:
+masquerade (the outgoing interface's address) or a fixed address with
+`--snat-to`. Like rules, they are in order and match by zone, address and
+service; the first that matches decides.
+
+```sh
+hc snat add --out wan --from trusted --comment "the LAN to the internet"
+hc snat add --out wg0 --not-out --from wg-clients
+hc snat add --out wan --from fd00:1::/48 --family ipv6 --snat-to 2001:db8::1
+```
+
+Source NAT is IPv4 only unless `--family ipv6` says otherwise: with a
+delegated prefix, IPv6 needs no NAT, and NAT66 is for unusual cases
+(a ULA network behind one public address).
+
+## Redirects
+
+A redirect sends matching traffic to a port on the firewall itself, for
+example to make every client use the firewall's DNS server:
+
+```sh
+hc redirect add --in lan --service dns --not-to 192.168.1.5 --comment "force DNS"
+hc redirect add --in lan --service http --to-port 3128
+```
+
+Without `--to-port`, the service's own port. The input chain must
+accept the redirected traffic: redirected DNS needs a rule accepting DNS
+from `lan`.

docs/operations.md

@@ -0,0 +1,335 @@
+# Running Housecarl
+
+## The services
+
+```sh
+systemctl status housecarl          # housecarld: the API, the web UI, the log
+journalctl -u housecarl -f
+systemctl reload housecarl          # reread the TLS certificate
+systemctl status housecarl-ruleset  # the ruleset loaded at boot
+```
+
+The firewall itself is the kernel's: it stays in force while housecarld
+is stopped, restarted or upgraded. Without housecarld there is no API,
+no web UI, no log and no probation timer.
+
+housecarld logs each change it applies, rollbacks, and what it couldn't
+apply. When it starts, it applies the configuration again (rolling back
+first what was on probation when it stopped).
+
+## Reviewing before applying
+
+In the web UI, edits are collected as pending changes (the bar under the
+header), and Changes shows them with the diff of the ruleset; Apply puts
+them in force together. The switch in the header (Review edits / Live
+edits) applies each edit at once instead. With hc, see
+[cli.md](cli.md#changesets).
+
+## Changes on probation
+
+Changes made over the network roll back unless confirmed
+([design.md](design.md#not-locking-yourself-out)); in shadow mode, where
+a change can't cut anyone off, none do ([shadow.md](shadow.md)). The web UI and `hc`
+confirm their own changes once they reach Housecarl again over a new
+connection, so normally nothing waits:
+
+```
+$ hc rule add input --from admins --service ssh
+created rule input #31
+applied and confirmed (history 12): this client reached Housecarl again over a new connection
+```
+
+A change that cuts the client off can't be confirmed that way:
+
+```
+applied on probation (history 13), but couldn't confirm it: … context deadline exceeded
+It is rolled back within 1m50s unless confirmed ('hc confirm')
+```
+
+By hand (with `--no-confirm`, or with Settings → "a person confirms each
+change"):
+
+```sh
+hc status              # Probation: 2 changes roll back in 97s unless confirmed
+hc confirm
+hc rollback            # or undo them now
+```
+
+The web UI shows a banner with a countdown and both buttons. A rollback
+is in the history, as an undo by `housecarl` via `rollback`, so it can be
+undone like any change.
+
+If you are cut off by a change: wait. When the time is up, the previous
+ruleset comes back by itself. If you are cut off by a confirmed change,
+use the console, or SSH from a source the rules still let in:
+
+```sh
+sudo hc history list
+sudo hc history undo 42
+```
+
+## The ruleset at boot
+
+A confirmed ruleset (or one changed through the socket) is saved to
+`/var/lib/housecarl/ruleset.nft`, which `housecarl-ruleset.service`
+loads before the network comes up, so before PPPoE, WireGuard or DHCP
+bring up their interfaces. To see what would load:
+
+```sh
+sudo cat /var/lib/housecarl/ruleset.nft
+sudo nft -c -f /var/lib/housecarl/ruleset.nft     # check it
+```
+
+In shadow mode both files are shadow mode's, and the fallback filters
+nothing ([shadow.md](shadow.md#what-works-differently)).
+
+Dynamic sets start empty after a reboot. Sets from Jarl have the
+addresses they last had, in the saved ruleset and again when housecarld
+starts, whether Jarl is reachable yet or not ([jarl.md](jarl.md#when-jarl-cant-be-reached)).
+The saved ruleset has no
+flowtable, since its devices may not exist yet: housecarld adds it when
+it starts, and whenever a flowtable device appears
+([flowtable.md](flowtable.md#devices-that-come-and-go)).
+
+### If the ruleset fails to load at boot
+
+Should `ruleset.nft` fail to load (a damaged file, an nftables or kernel
+upgrade that refuses something in it), the machine would otherwise boot
+with no firewall at all until housecarld starts. Instead,
+`housecarl-ruleset.service` loads a **fallback** that housecarld saves
+beside it, `/var/lib/housecarl/ruleset-fallback.nft`, from the same
+confirmed configuration:
+
+* **Nothing new gets in from the external zones' interfaces**, to the
+  firewall or through it: no port forwards, no SSH or web UI from the
+  internet.
+* What the firewall and the inside need to keep working is let in:
+  established connections, the ICMPv6 IPv6 needs (neighbour discovery,
+  router advertisements, packet too big), and DHCPv6 replies from
+  link-local addresses.
+* **Everything else is let through**: the LAN, WireGuard and the
+  firewall itself are reachable from the inside, so you can log in and
+  fix it.
+* IPv4 leaving an external zone is masqueraded (when any source NAT is
+  configured), so the inside still reaches the internet. MSS clamping
+  stays, if it is on.
+
+PPPoE itself isn't affected: its discovery and session are Ethernet
+frames, and the PPP link's negotiation happens before any IP, neither
+of which an `inet` table sees.
+
+The fallback only knows which interfaces face the internet from the
+zones marked **external**. With none, it lets nothing but loopback
+through, and the warning `no_external_zone` says so.
+
+When the fallback loads, the unit is marked failed, and the journal says
+why (as a critical message):
+
+```sh
+systemctl status housecarl-ruleset
+journalctl -b -u housecarl-ruleset
+```
+
+housecarld puts the real ruleset back when it starts, and saves a fresh
+`ruleset.nft`. If it can't apply it either, the fallback stays in force:
+`hc status` and the dashboard say "NOT in force: the boot fallback is in
+force", and `/healthz` answers 503 ([Monitoring](#monitoring)). Fix
+what `hc status` or the journal reports, then restart housecarld.
+
+Keep Debian's `nftables.service` disabled (`systemctl disable
+nftables`, without `--now`): at boot it loads `/etc/nftables.conf`, and
+stopping or restarting it runs `nft flush ruleset`, which deletes every
+table, Housecarl's included. `housecarl-ruleset.service` runs after it
+at boot, which covers the ordinary case, but not a restart of
+`nftables.service` later. Housecarl never reads or writes
+`/etc/nftables.conf`; its ruleset is saved in
+`/var/lib/housecarl/ruleset.nft`. The dashboard and `hc status` warn
+while `nftables.service` is enabled.
+
+Other programs' tables are left alone: fail2ban's (`inet f2b-table`),
+Docker's, Incus's. Housecarl replaces only its own table, so they keep
+their rules through every Housecarl change and restart, and nothing
+needs restarting in a particular order. A packet must get past every
+table on its hook, so a drop in any of them wins.
+
+## Hits
+
+Every rule, NAT rule, built-in and policy has a counter. `hc rule list`,
+`hc forward list` and the web UI show packets; `hc counters` shows them
+all, with bytes. They count from housecarld's start, and carry across
+changes to the ruleset. A deleted object's counter goes with it; an
+import with `--replace` gives every object a new ID, so its counters
+start again. A disabled rule keeps its count for when it is enabled.
+
+## Quotas
+
+`hc quotas` (and the rules page) show what each quota used and when it
+resets; `hc rule reset-quota ID` starts one again (`--address` one device's,
+`--give 5gbytes` a top-up until the next reset). housecarld resets
+quotas at midnight when their period ends (late, if it was stopped then),
+and saves their usage every five minutes, after every change and when
+it stops; after a reboot it puts it back. A reboot without a clean stop
+loses at most the last five minutes' count.
+
+## The log
+
+Rules with `--log` (Log in the web UI) and, with `hc settings set
+--log-drops on`, what the input and forward policies drop (at most 10
+packets a second) are logged:
+
+```sh
+hc log -f
+hc log --search 198.51.100.7
+```
+
+```
+10:02:13  hc input #95 drop:     wan  -   198.51.100.7:53868 -> 203.0.113.10:22   tcp syn
+10:02:15  hc forward policy drop: wan  eth1  198.51.100.9:443 -> 192.168.1.20:51432  tcp
+```
+
+In shadow mode ([shadow.md](shadow.md)) what the policies would drop is
+always logged, in all three chains, and drops are marked as what would
+have been done: the packets went on.
+
+The packets go to netlink log group 4171, where housecarld reads them;
+the last 2000 are kept in memory. They aren't written to the kernel log
+or the journal. A rule's log isn't rate-limited: on a busy rule, give it
+a limit, or log only new connections (`--state new`).
+
+## Connections
+
+The built-in that accepts established connections lets their packets
+through without checking the rules again: a rule added later doesn't
+stop a connection already open. Traffic → Connections (`hc connections`)
+lists what the kernel tracks, with NAT (`to 192.168.1.10:443`: a port
+forward; `as 203.0.113.10`: masqueraded) and the time left:
+
+```sh
+hc connections --search 192.168.1.50
+hc connections --search ":443 tcp"
+hc connections close --search 192.168.1.50
+```
+
+Closing a connection deletes its tracking entry: its next packet is
+checked by the rules as they are now. One they block is stopped; one they
+allow carries on (TCP connections are picked up again mid-stream).
+
+To make a block stop open connections by itself, tick "every packet" on
+the rule (`--every-packet`): it is checked before established connections
+are accepted, like rules with a quota ([rules.md](rules.md#rules)).
+
+## Tracing
+
+Traffic → Trace (`hc trace`) follows packets through the whole ruleset:
+Housecarl's rules and every other table (Docker's, fail2ban's, an old
+`inet main`), each rule and policy they meet, and what accepted or
+dropped them:
+
+```
+$ hc trace --address 192.168.1.50 --port 443 --seconds 60
+14:02:13.120  192.168.1.50:51432 -> 198.51.100.9:443  tcp syn in eth1 out wan
+    inet housecarl forward: built-in: mss-clamp -> continue
+    inet housecarl forward: rule forward #12 (kids at night) -> drop
+    => DROPPED by rule forward #12 (kids at night)
+```
+
+It traces packets from or to the address, on the port (either way), for
+5 to 600 seconds, at most 200 packets and 20 a second. It works with
+nftables' own tracing: a table of its own (`inet housecarl-trace`,
+hooked before everything else) marks the packets, and `nft monitor
+trace` reports them; the table goes when the trace ends. One trace runs
+at a time.
+
+## When something doesn't get through
+
+1. Trace it (above): it names the rule or policy that dropped it, in
+   whichever table.
+2. `hc log -f` with `--log-drops on`: is it dropped by a policy (no rule
+   accepted it), or by a rule (its number is in the line)?
+3. `hc rule list forward` (or `input`): hits on the rule you expect?
+4. `hc ruleset`: the rule as nftables sees it, with the zones' interfaces
+   and the sets' families.
+5. `hc status`: other tables. A Docker or Incus table, or an old one,
+   can drop what Housecarl accepts.
+6. Port forwards: `hc forward list` hits, and the inside host's own
+   firewall and default route (answers must go back through Housecarl).
+
+## Backups
+
+`housecarl-backup.timer` runs `housecarld backup` daily. It writes a
+consistent copy of the database to `backup.dir`
+(`/var/backups/housecarl/housecarl-<time>.db`), and deletes those older
+than `backup.retention` (30 days), keeping at least
+`backup.keep_at_least` (7). A backup holds everything: configuration,
+history, users, tokens (hashed) and the audit log.
+
+```sh
+sudo -u housecarl housecarld backup                 # now
+sudo -u housecarl housecarld backup /tmp/h.db       # to a file of your choice
+sudo systemctl disable --now housecarl-backup.timer # no daily backups
+```
+
+To **restore**, stop housecarld first; restore refuses to replace the
+database under it:
+
+```sh
+sudo systemctl stop housecarl
+sudo -u housecarl housecarld restore /var/backups/housecarl/housecarl-20261014T020000Z.db
+sudo systemctl start housecarl
+```
+
+housecarld applies the restored configuration when it starts. The old
+database is kept next to it as `housecarl.db.before-restore-<time>`.
+
+## Upgrades
+
+`apt upgrade` restarts a running housecarld. A new version upgrades the
+database when it starts, and an older one refuses a database a newer one
+has upgraded: restore a backup to go back.
+
+## Monitoring
+
+`GET /healthz` needs no login, and answers within a few seconds of any
+change (it reads the ruleset at most every five seconds, however often
+it is asked):
+
+```sh
+$ curl -s https://firewall.example.net:8070/healthz
+{"ok":true,"version":"0.1.0","firewall":"applied","mode":"enforcing","probation":0,"warnings":[]}
+```
+
+| Field | |
+|---|---|
+| `ok` | `true` with HTTP 200 when the firewall is as it should be; `false` with HTTP **503** otherwise |
+| `firewall` | `applied` (Housecarl's ruleset is in force), `unconfigured` (nothing configured yet, so nothing to apply), `fallback` (the [boot fallback](#if-the-ruleset-fails-to-load-at-boot) is in force), `missing` (no Housecarl table: flushed, or never applied) |
+| `mode` | `enforcing`, or `shadow`: Housecarl's ruleset is loaded but blocks nothing, and another firewall decides ([shadow.md](shadow.md)). `ok` doesn't depend on it; to be told when a machine isn't enforcing, also expect `"mode":"enforcing"` |
+| `probation` | changes waiting to be confirmed, rolled back unless they are |
+| `warnings` | codes of problems with the machine, as `hc status` shows them (`nftables_service`) |
+
+For Checkmk, an HTTP check on the URL that expects status 200 (and,
+if you like, the string `"firewall":"applied"`) covers housecarld being
+down, the fallback and a missing ruleset. Reach it from where the input
+rules let the monitor in (a rule for the Checkmk server's address and
+port 8070).
+
+## Failed logins in the journal
+
+A failed login to the web UI or the API, and a refused API token, are
+written to the journal as well as the audit log, with the address first:
+
+```
+level=WARN msg="login failed" ip=203.0.113.9 user=admin reason="wrong username or password"
+level=WARN msg="token refused" ip=203.0.113.9 reason="wrong token secret"
+```
+
+Guard's `housecarl` preset, or a fail2ban filter, can ban such addresses
+([bans.md](bans.md)). The address comes before the user name, which a
+caller chooses, so a pattern anchored on `msg="login failed" ip=` can't
+be fooled by a name. (With `log.format: json` they are JSON objects with
+the same fields, in the same order.)
+
+## Checks
+
+`housecarld check` validates the configuration in the database against
+this machine's interfaces, the same checks as `hc status` shows
+([validation.md](validation.md)).

docs/permissions.md

@@ -0,0 +1,96 @@
+# Users, permissions and tokens
+
+## Users
+
+**Administrators** may do everything, including managing users. Everyone
+else gets **grants**, `resource:access`:
+
+| Resource | read | write |
+|---|---|---|
+| `config` | settings, zones, sets, services, rules, NAT, the ruleset, history, interfaces, probation, changesets (the web UI shows the forms locked) | change them, stage and apply changesets (your own, or anyone's for administrators), undo, import, confirm and roll back; close connections; start and stop traces |
+| `sets` | dynamic sets' entries, what sets from Jarl hold | add and remove entries |
+| `bans` | ban lists' bans, and the lists the grant names | ban and unban ([bans.md](bans.md)) |
+| `log` | hit counters, the log, connections, traces | – |
+| `audit` | the audit log | – |
+| `*` | all of the above | |
+
+A `sets` grant can name the sets it covers: `sets:write:guests` may
+change the entries of the set `guests` and no other;
+`sets:read:guests,blocked` may read two. That is the grant for another
+program, such as Steward's captive portal, which should be able to let a
+client in and nothing more. A dynamic set's entries need a `sets` grant
+even for those with `config:write`.
+
+A `bans` grant names ban lists the same way: `bans:write:web` may ban
+into the list `web` (and read it) and nothing else, the grant for Guard
+or fail2ban on the web server.
+
+Only administrators may delete other tables (`hc takeover`), and manage
+users and the certificate (`hc cert`, Server → Certificate).
+
+**Roles** are ready-made sets of grants:
+
+| Role | Grants |
+|---|---|
+| `viewer` | `config:read sets:read bans:read log:read` |
+| `operator` | `config:write sets:write bans:write log:read audit:read` |
+| `log` | `log:read`, e.g. for a monitoring script |
+
+```sh
+hc user add sam --role operator
+hc user add portal --no-password --grant sets:write:guests     # tokens only
+hc user update sam --grant config:read --grant log:read
+```
+
+The last administrator can't be removed, demoted or disabled.
+
+## API tokens
+
+A token has its user's permissions, or fewer. A token made with `--role`
+or `--grant` can do only that, even if its user can do more:
+
+```sh
+hc token create portal --user portal --grant sets:write:guests --expires 365d
+```
+
+The secret is shown once; only its hash is stored. A restricted token can
+only create tokens with a subset of its own grants.
+
+In the web UI, Account → New token makes one of your own; an
+administrator makes one for another user with New token on that user's
+row under Users (`--user` in the CLI, `POST /users/{id}/tokens` in the
+API). The dialog only offers roles within what the token's user may do; a
+token with more would get no more than its user anyway.
+
+**Rotating** gives a token a new secret, keeping its ID, name and
+permissions, so only the programs that use it need updating:
+
+```sh
+hc token rotate 7 --grace 1h           # the old secret works for another hour
+hc token rotate --current --save       # this CLI's own token
+```
+
+A token that expires gets its lifetime again, from now (`--expires 90d`
+or `never` to change it). Without `--grace` the old secret stops at once:
+use that when it may have leaked. Using an old secret during its grace
+period is logged, naming the token and the address, so you can find the
+programs still to update. A restricted token may rotate itself and tokens
+no wider than itself. `hc token list` shows when each token was last
+used, and from where. `hc token revoke ID` stops one at once.
+
+## The local socket
+
+On the firewall, `/run/housecarl/housecarl.sock` lets root and the
+`housecarl` user act as administrator without a token, and their changes
+aren't on probation. The audit log shows them by their system user name.
+Both can already change the machine's firewall, so this adds convenience,
+not power. Anyone else on the socket needs a token, and their changes are
+on probation like any other.
+
+## The audit log
+
+Logins (and failed ones), users, tokens, every configuration change (with
+its history entry), changesets created, applied and discarded,
+confirmations and rollbacks, imports, takeovers,
+dynamic set entries and bans are recorded with who, from where and how: `session`,
+`token` or `socket`.

docs/rules.md

@@ -0,0 +1,457 @@
+# Zones, sets, services and rules
+
+## Zones
+
+A zone names one or more interfaces, so rules can say "from the LAN"
+rather than `eth1`. An interface belongs to one zone at most. A trailing
+`*` matches many: `incusbr*`.
+
+```sh
+hc zone add wan --interface wan --external
+hc zone add lan --interface eth1 --interface 'eth1.*'
+hc zone add wg  --interface wg0
+```
+
+**External** zones face the internet. Antispoofing, MSS clamping, the
+DHCPv6 client rule and port forwards (without public addresses) apply on
+them. Everything else is internal: ping to the firewall from internal
+zones is accepted by a built-in.
+
+A zone may name an interface that doesn't exist yet (a WireGuard
+interface that comes up later): its rules match once it appears, and
+`hc status` warns until then.
+
+**VLANs.** A VLAN sub-interface is an interface like any other, by its
+name: `enp2s0.20` (or whatever your network configuration calls it,
+`vlan20`, `br0.20`). Tagged traffic arrives on the sub-interface, and
+untagged traffic on the parent, so `enp2s0` alone doesn't cover
+`enp2s0.20`. Give each VLAN its own zone to keep them apart:
+
+```sh
+hc zone add lan --interface enp2s0          # untagged
+hc zone add iot --interface enp2s0.20
+hc zone add guest --interface enp2s0.30
+hc rule add forward --in iot --out wan
+```
+
+or one zone for the parent and all of its VLANs, when they should be
+treated alike: `--interface enp2s0 --interface 'enp2s0.*'`. Interface
+names are at most 15 characters (the kernel's limit). The flowtable
+takes VLAN sub-interfaces too, by exact name
+([flowtable.md](flowtable.md)).
+
+**Renaming** a zone (`hc zone update dhcp --name lan`, or Edit on the
+Zones page) changes every rule and NAT rule that names it, in the same
+change: the history shows them together, and undoing it puts them all
+back. Sets and services rename the same way, except dynamic sets: their
+entries, and the API tokens and programs that fill them, go by their
+name.
+
+## Address sets
+
+A set is a named list of IPv4 and IPv6 addresses, prefixes and ranges,
+mixed:
+
+```sh
+hc set add trusted 192.168.1.0/24 192.168.2.0/24 2001:db8:1::/64
+hc set add dns-servers 192.168.1.5 2001:db8:1::5 192.168.1.250-192.168.1.254
+```
+
+In the ruleset each becomes two nftables sets, `trusted_v4` and
+`trusted_v6` (one if the set has only one family). A rule that names it
+becomes one nftables rule per family. Rules can also give addresses
+directly (`--from 192.168.1.3`); a set is better when several rules share
+them.
+
+A **dynamic** set's entries are added at run time, each with a timeout,
+and nftables removes them when it runs out, even with housecarld stopped.
+It has no configured entries (validation refuses them); its timeout is
+the default for entries added without one, and without a timeout they
+stay until removed:
+
+```sh
+hc set add guests --dynamic --timeout 24h
+hc set entries add guests 10.0.30.17                 # for 24 hours
+hc set entries add guests 2001:db8:3::17 --timeout 2h
+hc set entries guests                                # with the time left
+hc set entries remove guests 10.0.30.17
+```
+
+Entries aren't configuration: they aren't in the history and aren't on
+probation, but each change is in the audit log. They survive every
+change to the ruleset, and a restart of housecarld; a reboot empties
+them. A token can be limited to one set ([permissions.md](permissions.md)):
+that is how another program (Steward's captive portal) lets a client in.
+
+A set **from Jarl** holds the addresses of names in Jarl, the BIND
+manager: every A and AAAA record of each name, CNAMEs followed, read again
+every minute. Only addresses within the zones and networks it names are
+used, so a changed record can't open the firewall to anything else:
+
+```sh
+hc set add nas --jarl nas.example.com@internal --within lan
+hc set jarl nas                                      # what it holds, and why
+```
+
+Views, CNAMEs, what happens when Jarl can't be reached and the rest are in
+[jarl.md](jarl.md#sets-from-jarl).
+
+## Services
+
+A service names ports of a protocol (`tcp`, `udp`, `sctp`) and ICMP
+types:
+
+```sh
+hc service add web tcp/8080 tcp/8443
+hc service add games udp/27015-27030 tcp/27015
+hc service add pmtu icmp/destination-unreachable icmpv6/packet-too-big
+```
+
+Many are built in (`hc service list`): `ssh`, `dns` (TCP and UDP 53),
+`dot`, `http`, `https` (TCP, and UDP for QUIC), `smtp`, `submission`,
+`smtps`, `imaps`, `ntp`, `dhcp`, `dhcpv6`, `wireguard`, `openvpn`, `ipsec`,
+`snmp`, `syslog`, `mdns`, `rdp`, and `ping` (ICMP and ICMPv6 echo
+requests). Adding a service with a built-in's name replaces it. Rules may
+also give an entry directly: `--service tcp/8080`.
+
+## Rules
+
+Rules are kept in order, in four chains:
+
+| Chain | | Hook |
+|---|---|---|
+| `input` | traffic to the firewall itself | input |
+| `forward` | traffic through it, between zones | forward |
+| `output` | traffic from it | output |
+| `early` | before connection tracking: cheap drops of what can never be right | prerouting, priority raw |
+
+Each chain is checked top to bottom, and the first rule that accepts,
+drops or rejects decides. What no rule decides gets the chain's
+**policy** (Settings): by default input and forward drop, output
+accepts.
+
+```sh
+hc rule add input --from trusted --service ssh --comment "SSH from home"
+hc rule add input --in wg --service dns
+hc rule add forward --in lan --out wan
+hc rule add forward --in wan -a drop --comment "nothing else gets in"
+hc rule list input
+hc rule move 12 --before 7
+```
+
+A rule matches on what it gives, and all of it:
+
+| | |
+|---|---|
+| `--in`, `--out` | zones it comes in on, goes out on (`--not-in`, `--not-out`: any other). Input has no out zone, output no in zone |
+| `--from`, `--to` | sets, addresses, prefixes, ranges (`--not-from`, `--not-to`: anything else) |
+| `--service` | services or entries (`tcp/8080`, `icmp/echo-request`) |
+| `--family` | `ipv4` or `ipv6` only; otherwise it follows the addresses |
+| `--state` | connection states: `new`, `established`, `related`, `invalid`, `untracked` |
+| `--status` | connection status: `dnat` (port forwarded), `snat` |
+| `--days`, `--hours` | a time window: `--days Monday,Friday --hours 08:00-17:30`; `22:00-06:00` passes midnight. In the firewall's time zone, as the kernel knows it |
+| `--limit` | a rate: `10/minute`, or bytes: `1mbytes/second`; with `,burst=N`, `,over` (match beyond the rate) and `,per-source` or `,per-destination` ([Bandwidth and quotas](#bandwidth-and-quotas)) |
+| `--quota` | an allowance of traffic: `50gbytes,over,per-source,monthly=15` ([Bandwidth and quotas](#bandwidth-and-quotas)) |
+| `--conn-limit N` | a source with more than N connections |
+
+The action is `accept`, `drop`, `reject` (an ICMP error, or with
+`--reject-with tcp-reset` a TCP reset) or `log` (log and go on).
+`--log` logs the packets of any rule ([operations.md](operations.md#the-log)).
+`--disabled` (or `hc rule disable ID`) keeps a rule without using it.
+`--every-packet` ("every packet" in the web UI) checks a drop or reject
+against every packet, before established connections are accepted, so it
+also stops connections already open, not just new ones; it can't accept
+([operations.md](operations.md#connections)).
+
+Examples:
+
+```sh
+# SSH from the internet, at most 3 new connections a minute per source
+hc rule add input --in wan --service ssh --state new --limit 3/minute,burst=5,over,per-source -a drop --log
+hc rule add input --in wan --service ssh
+
+# Children's devices offline at night
+hc rule add forward --from kids --out wan --hours 22:00-07:00 -a reject
+
+# What was port forwarded may go on, the rest from the internet may not
+hc rule add forward --in wan --status dnat
+hc rule add forward --in wan -a drop
+```
+
+### Matching everything but a few
+
+A field matches **any of** its entries, or with `--not-…` ("any but" in
+the web UI) anything else; it can't do both. "The LAN except the NAS" is
+still one rule: the zone says which network, and the addresses say which
+of it to leave out, because zones and addresses are different fields and
+a rule needs all of them to match.
+
+| What you want | One rule |
+|---|---|
+| the LAN, except the NAS | `--in lan --from nas --not-from` |
+| everyone on guest, except one device, has a quota | `--in guest --from tv --not-from --quota 10gbytes,over,daily` |
+| the internet, except one host | `--out wan --to blocked --not-to` |
+
+```sh
+# Each device on the LAN: 50 GB a month, but not the NAS (its backups)
+hc rule add forward --in lan --out wan --from nas --not-from --quota 50gbytes,over,per-source,monthly -a drop
+```
+
+Leaving part of an address range out ("10.0.0.0/8 except 10.0.5.0/24")
+when it isn't a zone takes one of:
+
+* **Two rules, the exception first**, when the exception gets the
+  opposite action: `--from 10.0.5.0/24 -a drop` above `--from
+  10.0.0.0/8`. There is no action that skips a rule and goes on, so this
+  doesn't work when the exception should just fall through to later
+  rules.
+* **Ranges around the hole**: `--from 10.0.0.0-10.0.4.255 --from
+  10.0.6.0-10.255.255.255`.
+* **A [raw rule](#raw-rules)**: `ip saddr 10.0.0.0/8 ip saddr != 10.0.5.0/24 accept`.
+
+There's no need to leave a zone out of other zones: an interface belongs
+to one zone, so "any of lan, guest" already leaves out everything else.
+
+### Bandwidth and quotas
+
+A **byte rate** limits bandwidth: `--limit 20mbytes/second,over,per-source
+-a drop` drops what a device sends beyond 20 MB/s. Units are `bytes`,
+`kbytes` and `mbytes` (`kb`, `mb` for short); the burst is in the same
+unit. nftables can only hold byte rates per second (up to 4 GB/s) or per
+minute (up to 293 MB/minute); validation says so for larger ones. This is
+policing: traffic beyond the rate is dropped, not queued, so TCP slows to
+fit, and UDP loses packets. Shaping (queues, fair shares) is `tc`'s work,
+not a firewall's.
+
+A **quota** is an allowance of traffic. `over` matches once it is used up,
+to drop what comes after:
+
+```sh
+# Each device on the LAN: 50 GB a month, from the 15th (a billing day)
+hc rule add forward --in lan --out wan --quota 50gbytes,over,per-source,monthly=15 -a drop
+# The whole guest network: 10 GB a day together
+hc rule add forward --in guest --out wan --quota 10gbytes,over,daily -a drop
+hc quotas                      # what each used, and the next reset
+hc rule reset-quota 31         # start again now
+hc rule reset-quota 31 --address 192.168.1.23            # one device only
+hc rule reset-quota 31 --give 5gbytes --address 192.168.1.23
+```
+
+`--give` is a top-up: it takes that much off what was used (never below
+zero), so the device gets 5 GB more until the quota's next reset, and the
+rule stays as it is. In the web UI, the quota's line under the rule
+opens its dialog: usage (each address's), resets, and **Reset** and
+**Give back…** for everyone or one address. Like a reset, these are
+done at once (audited, not staged or on probation); Housecarl puts the
+ruleset in force again with the new figure.
+
+In the web UI each quota's use is on its rule (with **Reset**), all of
+them together on the dashboard, and their history under **Traffic →
+Statistics**.
+
+| | |
+|---|---|
+| `per-source`, `per-destination` | each address its own allowance; without, one allowance for everything the rule matches |
+| `daily`, `weekly`, `monthly=DAY` | when it starts again: at midnight; weekly on Mondays; monthly on that day (the last day of a shorter month). Without one, only by hand |
+
+nftables counts a quota but never resets it, and forgets it when its
+table is replaced: Housecarl carries what each quota used across every
+change and restart, saves it every five minutes (and after each change,
+and when housecarld stops) for after a reboot, and resets quotas when their period ends.
+A per-address quota counts an address: a device that gets a new one
+starts again. With Steward's reservations IPv4 addresses stay; IPv6 is
+grouped by /64 (below).
+
+Changing the amount, `over` or the resets keeps what was used. Changing
+how the quota counts carries what can be carried:
+
+| Change | What was used |
+|---|---|
+| per address to one total | the addresses' sum |
+| to wider groups (IPv6 /64 to /56) | added up per new group |
+| a total to per address | starts again: a total can't be split |
+| per source to per destination (or back) | starts again |
+| to finer groups (/64 to /128) | starts again: a group can't be split |
+
+A change that starts a quota again says so (warning `quota_restarts`),
+when it is staged for review and in `hc` (also with `-n`).
+
+**Rules with a quota or a byte rate count every packet.** The built-in
+that accepts established connections comes before your rules, so a rule
+there only sees each connection's first packet. Rules that count traffic
+are checked before it instead, in their order (the web UI lists them
+first, marked "every packet"). So they can drop, reject or log, but not
+accept: an accept there would let packets past every later rule. Let the
+traffic in with a rule of its own. The flowtable's fast path skips the
+forward chain for established connections, so with it on, forward
+quotas and byte rates only count first packets (a warning says so;
+[flowtable.md](flowtable.md)).
+
+**Per-address limits group addresses**: each IPv4 address on its own, and
+IPv6 addresses by /64, since one device uses several (privacy addresses
+change). `--group-v4 24` counts each /24 together; `--group-v6 128` each
+IPv6 address alone. This applies to per-address rates, connection limits
+and quotas.
+
+### Raw rules
+
+For what Housecarl can't say, a rule can be one nftables rule written by
+hand, placed like any other:
+
+```sh
+hc rule add forward --raw 'iifname != "wan" ct state invalid ip daddr @trusted_v4 accept'
+```
+
+It is used exactly as written: its text replaces the rule's zones,
+addresses, services, action, limits and log, so a raw rule with any of
+those set is refused (`raw_conflict`); write them in the text. It is
+checked by nftables with the rest of the ruleset, and can use the sets'
+nftables names (`@trusted_v4`). The importer makes raw rules from
+what it can't translate. It goes into the ruleset as written: add
+`counter` to it to count its packets.
+
+In shadow mode ([shadow.md](shadow.md)) a raw rule's final `drop` or
+`reject` becomes `accept`; a raw rule that does anything else to packets
+(NAT, `meta mark set`, `jump`, `notrack`, a verdict map…) is left out,
+with a warning (`shadow_raw_left_out`).
+
+## Ban lists
+
+Programs such as Guard and fail2ban ban addresses into ban lists, and
+those bans come first of all in the input and forward chains (after
+loopback in input), before the built-ins and every rule. See
+[bans.md](bans.md).
+
+## Built-ins
+
+Housecarl adds the rules every firewall needs before yours. Each can be
+turned off (`hc settings set --builtin NAME=off`; Settings in the web
+UI), and `hc ruleset` shows them:
+
+| Built-in | Default | Chains | What |
+|---|---|---|---|
+| `loopback` | on | input (and output if its policy is drop) | accept everything on `lo` |
+| `established` | on | input, forward (and output) | accept established and related connections |
+| `drop_invalid` | on | input, forward | drop packets connection tracking calls invalid |
+| `icmpv6` | on | input, forward (and output) | the ICMPv6 IPv6 can't work without (RFC 4890): errors and echo; neighbour discovery with hop limit 255 and MLD from link-local, to the firewall |
+| `ping_internal` | on | input | ping (ICMP echo) to the firewall from internal zones |
+| `dhcpv6_client` | on | input | DHCPv6 replies from link-local on external zones, so the firewall gets its prefix delegation |
+| `antispoof` | on | early | drop sources that can't come from the internet, on external zones: the antispoof lists (private, loopback, documentation and multicast ranges) and your own prefixes |
+| `mss_clamp` | off | forward | clamp TCP's MSS to the route's MTU on external zones: needed with PPPoE |
+
+**Your prefixes** (`--own-prefix 2001:db8::/48`) are your delegated
+IPv6 prefix and public addresses: nothing from outside should come from
+them. The antispoof lists can be changed with `--antispoof-v4` and
+`--antispoof-v6`.
+
+The **flowtable** (`--flowtable wan --flowtable eth1`) lets established
+TCP and UDP connections forwarded between those devices skip the rules,
+which helps when the CPU can't keep up with the line. It needs exact
+device names, and costs quotas, byte rates and traffic per device in the
+forward chain their accuracy: see [flowtable.md](flowtable.md) for when
+to use it, and when not.
+
+## The zone matrix
+
+The Matrix page (`hc matrix`) sums up what the rules do to **new
+connections** between zones: a row per zone the traffic comes from, a
+column per zone it goes to, and the firewall itself (to it: the input
+chain; from it: output; zone to zone: forward).
+
+```
+$ hc matrix
+FROM \ TO  guest         lan           wan         firewall
+guest      block *       block         partly +1   partly +4
+lan        block         block *       allow       partly +4
+wan        partly +1 -1  partly +1 -1  block -1 *  partly +3 -1
+firewall   allow         allow         allow       -
+
+Let through for part of the traffic:
+  guest → wan       http, https
+  guest → firewall  ICMPv6; ping; ssh from admins; dns
+  lan → firewall    ICMPv6; ping; ssh from admins; dns
+  wan → guest       1 port forward
+  wan → lan         1 port forward
+  wan → firewall    ICMPv6; DHCPv6; ssh from admins
+
+allow, block: every new connection; partly: some are let through, some not.
++N: rules that let part of the traffic through; -N: rules that block part of it. 'hc matrix FROM TO' lists them.
+*: a zone to itself: only traffic routed between its interfaces passes the firewall.
+```
+
+Each cell walks that pair's rules in order, as nftables would, with the
+built-ins that act on new connections (ping from internal zones,
+ICMPv6, the DHCPv6 client, antispoofing, port forwards):
+
+* a rule that applies to the whole pair decides it;
+* a rule that applies to part of its traffic (some addresses, services,
+  times, rates, a quota) is an **exception**, and the walk goes on;
+* what's left falls to the deciding rule or the chain's policy.
+
+A cell reads **allow** or **block** when every new connection gets the
+same, and **partly** when exceptions let some through while the rest is
+blocked (or block some while the rest is allowed). **+N** counts the
+exceptions that let traffic through, **-N** those that block it. Under
+the grid (in the web UI, in each cell) is what the exceptions let
+through: their services, with the addresses they are limited to.
+
+A zone to itself (`*`, faded in the web UI) is mostly traffic that never
+reaches the firewall: within one interface's network it goes straight
+from host to host. Only what is routed between the zone's interfaces (or
+back out of the one it came in on) passes.
+
+`hc matrix guest wan` (or clicking the cell) lists the exceptions in
+order, with each rule's hits:
+
+```
+guest → wan (the forward chain), new connections: partly allowed
+  VERDICT  WHAT         BY                HITS
+  allowed  http, https  rule forward #14  1520
+  blocked  the rest     forward policy    37
+```
+
+The hits are the rule's packets since housecarld started, in all of its
+traffic: a rule without zones counts for every pair it covers. They are
+how you tell an exception that is used from one that isn't.
+
+A raw rule is read when it is a match followed by `accept`, `drop` or
+`reject` (with `counter` and `log` if you like): it counts as an exception
+for every pair of its chain, shown as `raw: …`. One that matches
+interfaces (the matrix can't tell which zones), jumps, translates, sets
+something or uses a verdict map shows as **unknown**: check it yourself.
+Raw rules have hits only if they say `counter`.
+
+A rule whose addresses all lie in networks directly connected to other
+zones is left out of a pair: traffic between those zones can't have them
+(`from wg_net` doesn't concern traffic coming in on the LAN). Routed
+networks, mixed lists and dynamic sets might, so they stay. Replies and
+established connections aren't shown: the built-in lets them through. In
+review mode, the matrix shows your pending changes.
+
+## The ruleset
+
+`hc ruleset` prints what Housecarl builds; the web UI and `hc rule list`
+show each rule's hits. The table looks like this (shortened):
+
+```
+table inet housecarl {
+	set trusted_v4 { type ipv4_addr; flags interval; elements = { 192.168.1.0/24, 192.168.2.0/24 } }
+	set trusted_v6 { type ipv6_addr; flags interval; elements = { 2001:db8:1::/64 } }
+	chain early   { type filter hook prerouting priority raw; policy accept; ... }
+	chain input   { type filter hook input priority filter; policy drop;
+		iifname "lo" accept comment "hc:b-loopback"
+		ct state established,related counter accept comment "hc:b-established"
+		...
+		ip saddr @trusted_v4 tcp dport 22 counter accept comment "hc:r4 SSH from home"
+		ip6 saddr @trusted_v6 tcp dport 22 counter accept comment "hc:r4 SSH from home"
+		counter comment "hc:p-input"
+	}
+	chain forward { ... }
+	chain output  { ... }
+	chain dstnat  { type nat hook prerouting priority dstnat; ... }
+	chain srcnat  { type nat hook postrouting priority srcnat; ... }
+}
+```
+
+Other tables still apply: a packet must get past every table's chains on
+its hook. The dashboard and `hc status` list them.

docs/shadow.md

@@ -0,0 +1,174 @@
+# Shadow mode
+
+In shadow mode Housecarl runs beside the firewall you have now and
+changes nothing. Its ruleset is loaded, so its counters, log, tracer and
+statistics show what it *would* do with your real traffic, but it blocks,
+rejects and translates nothing. Your current firewall keeps deciding.
+
+Use it to:
+
+* **try Housecarl** on the real machine before you trust it with the
+  firewall: the web UI, the API, `hc`, changesets, history;
+* **check an import**: import your ruleset ([import.md](import.md)) and
+  watch whether Housecarl would drop anything your firewall lets through;
+* **develop against the API**: Steward, scripts or fail2ban can fill
+  dynamic sets and see them used, with nothing at stake.
+
+## Turning it on
+
+Shadow mode is set in `/etc/housecarl/housecarl.yaml`
+([configuration.md](configuration.md)), not through the API, so no user
+or token can switch enforcement off:
+
+```yaml
+shadow:
+  enabled: true
+  position: after     # or before; see below
+```
+
+housecarld reads the file when it starts:
+
+```sh
+sudo systemctl restart housecarl
+sudo hc status        # Mode: SHADOW …
+```
+
+The web UI shows a banner on every page, `hc status` a `Mode:` line, and
+`/healthz` `"mode": "shadow"`.
+
+## Why it can't let anything through
+
+In nftables, `accept` only ends the chain it is in. The packet then
+goes on to the next base chain on the same hook, and other tables' chains
+are among those. Only a drop is final. Housecarl's table in shadow mode
+never drops, so it can't let through anything your firewall drops, and it
+can't block anything your firewall accepts.
+
+## What changes in the ruleset
+
+The same configuration is rendered, with everything that acts on packets
+turned into counting:
+
+| | Enforcing | Shadow mode |
+|---|---|---|
+| Rules that drop or reject | `drop`, `reject` | counted, then `accept` (Housecarl's chain ends as it would have). Their comment says `[would drop]` or `[would reject]` |
+| The built-ins that drop (invalid packets, antispoofing) | `drop` | counted, `[would drop]` |
+| Policies | as configured | `accept`. What a drop policy would drop is counted and **always logged**, rate-limited, in all three chains |
+| Port forwards, source NAT, redirects | translate | counted only (`[would dnat]`, `[would masquerade]`, `[would snat]`, `[would redirect]`) |
+| MSS clamping | changes SYN packets | left out |
+| The flowtable | offloads connections | left out: offloaded connections skip *every* table's forward chain, your firewall's too |
+| Raw rules | as written | as written if they only match, count, log or accept. A final `drop` or `reject` becomes `accept`. Anything else that acts on packets (NAT, `set`, `jump`, `notrack`, a verdict map…) is left out, with a warning (`shadow_raw_left_out`) |
+| Everything else | | the same: accepts, rate limits, connection limits, quotas, dynamic sets, per-device counters, logging |
+
+`hc ruleset` prints the shadow ruleset, the one loaded now.
+`hc ruleset --enforcing` prints the ruleset that will be loaded once
+shadow mode is off.
+
+**Every change is checked by nftables twice**: the shadow ruleset, and the
+enforcing one. A change that nftables would refuse once shadow mode is off
+is refused now, so leaving shadow mode can't fail on that.
+
+## Where Housecarl's chains run
+
+Your firewall's chains are usually at priority `filter` (0), its raw
+chains at `raw`. `shadow.position` puts Housecarl's filter chains just
+next to them:
+
+* **`after`** (the default) runs them at `filter + 10` (and `raw + 10`).
+  Housecarl only sees what your firewall let through. So **every would-drop
+  is traffic that would break if you switched over**: the list to work
+  through before you do. What your firewall drops and Housecarl would
+  accept (a hole) doesn't show; compare the rules for that.
+* **`before`** runs them at `filter - 10` (and `raw - 10`). Housecarl sees
+  every packet, as it would when enforcing, and its hits and drops are the
+  numbers you would get. Its would-drops include everything your firewall
+  drops too (the internet's background noise).
+
+If your firewall uses other priorities, say `filter + 20`, "after" is
+relative to `filter`, not to it: check with `nft list ruleset`.
+
+NAT chains are different: the kernel asks NAT chains in turn only until
+one translates. Housecarl's NAT chains always run first (`dstnat - 10`,
+`srcnat - 10`), whatever the position, or they would never see a
+connection your firewall translates. They translate nothing, so your
+firewall's NAT still does the work.
+
+## What you see
+
+* **Hits** (`hc counters`, the Rules page): a drop rule's hits are what it
+  would have dropped. NAT counters count connections (only a connection's
+  first packet reaches the NAT chains).
+* **The log** (`hc log`, the Log page): every packet a drop policy would
+  have dropped, and what rules with logging would have dropped, marked
+  *would drop*. `hc log` adds "(shadow mode: would drop, let through)".
+* **The tracer** (`hc trace`, the Trace page): the verdict is your
+  firewall's, the real one. When Housecarl would have dropped the packet,
+  the trace says by which rule ("shadow mode: Housecarl would have
+  dropped it, by …").
+* **Statistics** ([stats.md](stats.md)): drops, drops per chain and the
+  blocked sources are what Housecarl would have dropped.
+
+## What works differently
+
+* **No probation.** A change can't cut anyone off, so changes from the
+  network are applied and saved at once, with no confirmation. Changesets
+  work as usual; their ruleset diff is of the shadow ruleset.
+* **No takeover, no closing connections.** `hc takeover` (deleting your
+  firewall's tables) and `hc connections close` are refused (409,
+  `hc` exits with 5): both would change your traffic.
+* **Bans** ([bans.md](bans.md)) are kept and their rules count what they
+  would drop (`[would drop]`), but nothing is dropped, and banning closes
+  no connections.
+* **The boot.** `housecarl-ruleset.service` loads the shadow ruleset at
+  boot. Its fallback ([operations.md](operations.md#if-the-ruleset-fails-to-load-at-boot))
+  is an empty table that filters nothing, since your firewall protects the
+  machine.
+* **Your firewall reloading.** A ruleset file such as `/etc/nftables.conf`
+  usually starts with `flush ruleset`, and Debian's `nftables.service`
+  runs `nft flush ruleset` when it stops or restarts. Both delete
+  Housecarl's table too. housecarld watches nftables' notifications and
+  puts its table back within a few seconds (the journal says "Housecarl's
+  table was deleted … put it back"). Dynamic sets lose their entries, and
+  quotas go back to what was last saved (every five minutes).
+* **Warnings.** `nftables_service` and `takeover_pending` aren't raised:
+  another firewall beside Housecarl is the point. Instead:
+  `shadow_unprotected` when no other table has a filter chain on the input
+  hook (nothing protects the machine), and `shadow_raw_left_out` for each
+  raw rule left out ([validation.md](validation.md#the-machine)).
+* **Quotas count** what passes in shadow mode, and keep counting when
+  you switch over.
+
+## A trial, step by step
+
+From a machine whose firewall is `/etc/nftables.conf` with `table inet main`:
+
+1. Install Housecarl and create the administrator
+   ([install.md](install.md)).
+2. Turn shadow mode on in `/etc/housecarl/housecarl.yaml`, and start
+   housecarld: `sudo systemctl enable --now housecarl`.
+3. Import your ruleset: `sudo hc import --defines /etc/nftables.conf`
+   ([import.md](import.md)). Leave `nftables.service` as it is.
+4. Use your network for a while, and watch `hc log -f`, `hc counters` and
+   the dashboard. With `position: after`, each would-drop is something
+   your firewall allows and Housecarl wouldn't: fix the rule in Housecarl,
+   or decide it doesn't matter. `hc status` lists the raw rules left out.
+5. When nothing you need would be dropped, leave shadow mode.
+
+## Leaving shadow mode
+
+1. Look at what will be loaded: `sudo hc ruleset --enforcing`.
+2. Set `shadow.enabled: false` (or remove the section), and
+   `sudo systemctl restart housecarl`. Housecarl's ruleset is now in
+   force **alongside** your firewall: a packet must be accepted by both.
+   The restart applies it at once, not on probation, but your old firewall
+   is still there and still decides too.
+3. Take over: `sudo hc takeover "inet main"` deletes the old table (on
+   probation over the network), and `sudo systemctl disable nftables`
+   (without `--now`) retires it ([import.md](import.md#3-take-over)).
+
+## Going back
+
+Set `shadow.enabled: true` and restart housecarld. If Housecarl was your
+only firewall, **the machine is now unprotected**: load your other
+firewall first. Until one filters the input hook, `hc status` and
+`/healthz` warn (`shadow_unprotected`).

docs/stats.md

@@ -0,0 +1,144 @@
+# Statistics
+
+Housecarl keeps what the firewall did over time:
+
+- traffic per interface and zone, received and sent;
+- packets dropped, by chain, and when drops happen (a weekday × hour
+  heatmap);
+- hits per rule, built-in and policy, and the rules that matched nothing;
+- the sources blocked most and what they tried (from the log);
+- traffic per inside device (a setting, below);
+- connections tracked, against the kernel's limit;
+- what each quota counted;
+- housecarld's and the machine's memory, CPU and load.
+
+All of it is in the web UI (**Traffic → Statistics**, and two charts on
+the dashboard), the CLI (`hc stats`) and the API (`/api/v1/stats/...`),
+for anyone with `log:read`.
+
+Statistics are **on by default**. They cost nothing on the packets'
+path: housecarld reads counters the kernel keeps anyway, once a minute.
+Only traffic per device adds to the ruleset, so it is a setting you turn
+on. The figures are kept as aggregates in a database of their own,
+`/var/lib/housecarl/stats.db`, separate from the configuration. Deleting
+it is always safe (stop housecarl first), and backups leave it out.
+
+## Settings
+
+In `/etc/housecarl/housecarl.yaml` (restart housecarl after a change):
+
+```yaml
+stats:
+  enabled: true
+  database: /var/lib/housecarl/stats.db
+  retention:
+    minute: 7d     # per-minute figures
+    hour: 90d      # hourly figures and top lists
+    day: 730d      # daily figures and top lists
+  top_n: 100       # entries kept per top list, per hour and day
+```
+
+Old data is removed once an hour. A range picks the finest data that
+covers it: per minute up to 2 days, hourly up to 90 days, then daily.
+
+## Traffic per device
+
+```sh
+hc settings set --stats-per-device on
+```
+
+(or **Settings → Statistics** in the web UI). This is a firewall setting,
+so it goes through the usual review and probation. Housecarl then adds
+four sets to its table, and four rules at the top of the forward chain
+(shown as the built-in `stats`):
+
+```
+iifname != "wan" oifname "wan" update @stats_up_v4 { ip saddr }
+iifname != "wan" oifname "wan" update @stats_up_v6 { ip6 saddr and ffff:ffff:ffff:ffff:: }
+iifname "wan" oifname != "wan" update @stats_down_v4 { ip daddr }
+iifname "wan" oifname != "wan" update @stats_down_v6 { ip6 daddr and ffff:ffff:ffff:ffff:: }
+```
+
+Each inside address talking to an external zone gets a counter: what it
+sent out, and what came back to it. IPv6 addresses are counted per /64,
+since one device uses many addresses. Counting is done in the kernel, a
+set lookup per forwarded packet. An address quiet for an hour drops out
+of the sets, and a reload keeps the counts.
+
+With the flowtable on, established connections skip the forward chain,
+so only their first packets are counted (warning `stats_flowtable`). Set
+names starting with `stats_` (and `meter_`, `conn_`, `quota_`) are
+Housecarl's own and can't be used for your sets.
+
+## What is collected
+
+Every minute housecarld samples its sources and stores what changed:
+
+| Figure | Source |
+|---|---|
+| `rx:IF`, `tx:IF` (bytes), `rxp:IF`, `txp:IF` (packets) | the interfaces' counters (`/sys/class/net`); the loopback is left out |
+| `zrx:ZONE`, `ztx:ZONE` | the same, summed per zone: received from and sent to the zone's interfaces |
+| `hits:MARK`, `hitb:MARK` | each rule's (`r12`), built-in's (`b-established`), port forward's (`f3`) and policy's (`p-input`) counter, in packets and bytes |
+| `drop:CHAIN`, `drops` | the packets counted by what drops them: rules that drop or reject, the drop policies, antispoofing (`early`) and invalid packets (`invalid`) |
+| `quota:r12` | bytes counted against a rule's quota |
+| `g:conntrack`, `g:conntrack_max` | connections tracked, and the limit (`nf_conntrack_max`) |
+| `g:rss`, `g:cpu` | housecarld's memory (bytes) and CPU (tenths of a percent of one CPU) |
+| `g:mem`, `g:load` | the host's memory in use, and its 1-minute load average × 100 |
+
+`g:` figures are gauges: the API averages them over a bucket.
+
+Top lists, per hour and per day, the `top_n` largest:
+
+| List | |
+|---|---|
+| `blocked_src` | sources of logged drops |
+| `blocked_port` | what they tried: `tcp/22`, `udp/53`, `icmp/echo-request` |
+| `device_up`, `device_down` | bytes per inside address, sent and received (traffic per device) |
+| `quota_r12` | a per-address quota's use, per address |
+
+The blocked lists only see what is **logged**: the input and forward
+policies' drops with `hc settings set --log-drops on`, and rules with
+logging. The drop counts above see everything.
+
+In shadow mode ([shadow.md](shadow.md)) the drops, and the blocked
+lists, are what Housecarl would have dropped: the packets went on. What
+the policies would drop is always logged there.
+
+A counter that goes down (an interface that came back, a quota reset)
+counts again from zero. What happens while housecarld is stopped isn't
+counted.
+
+## The CLI
+
+```sh
+hc stats --since 7d              # summary: traffic per zone, drops, busiest rules, top lists
+hc stats status                  # collecting? since when, the database, figures now
+hc stats traffic --since 24h     # per zone over time (--interfaces: per interface)
+hc stats top devices --since 30d # devices by traffic (blocked, tried, quota RULE-ID)
+hc stats rules --unused          # rules that matched nothing in 30 days
+hc stats connections --since 24h
+```
+
+Ranges are written `90m`, `24h`, `7d` or `4w`. `-o json` gives the API's
+answer.
+
+## The API
+
+| Endpoint | |
+|---|---|
+| `GET /stats/status` | `enabled`, `database`, `size`, `first_data`, `last_sample`, `sample_error`, `per_device`, `retention`, `latest` (the newest gauges) |
+| `GET /stats/summary?since=24h&limit=10` | `zones` and `interfaces` (`name`, `received`, `sent`), `drops`, `drops_by_chain`, `rules` (busiest, named), `peak_connections`, `conntrack_max`, `blocked_sources`, `blocked_ports`, `devices_up`, `devices_down` |
+| `GET /stats/timeline?keys=zrx:*,drops&since=24h&step=auto` | `step`, `times` (bucket starts), `series` (`key`, `values`); `GROUP:*` means every key of a group; `step` is `auto`, `1m`, `5m`, `15m`, `1h`, `6h` or `1d`; a gauge without a sample is -1 |
+| `GET /stats/top?kind=device_down&since=7d&limit=20` | a top list (`key`, `count`) |
+| `GET /stats/hits?since=30d` | packets per rule, built-in and policy (`key`, `count`, `name`), with the enabled rules that counted none |
+| `GET /stats/heatmap?key=drops&since=28d` | `avg[weekday][hour]` (Monday first, local time; -1 where there is no data) |
+
+With statistics off, `/stats/status` says `enabled: false` and the others
+answer 503.
+
+## Warnings
+
+When the connection tracking table is over 80% full, the status shows
+the warning `conntrack_full`: once it is full, the kernel drops new
+connections. Raise `net.netfilter.nf_conntrack_max`, or find what opens
+so many on the Connections page.

docs/validation.md

@@ -0,0 +1,163 @@
+# Validation
+
+Every change is checked as a whole configuration before it is applied.
+An **error** stops the change; a **warning** is reported with it, and
+stays on the dashboard and in `hc status` until fixed. Each problem has a
+stable code, shown by the CLI as `[code]` and in the API as
+`problems[].code`. `housecarld check` runs the checks on the saved
+configuration.
+
+After Housecarl's own checks, **nftables** checks the rendered ruleset
+(`nft -c`). What it refuses (mostly raw rules) comes back as a
+`firewall` error with nftables' message, and nothing is changed.
+
+The warnings marked *host* compare the configuration with this machine's
+interfaces.
+
+## Names
+
+Zones, sets, services and port forwards have names (a forward's is
+optional): a letter, then letters, digits, `-` and `_`, up to 48. Names
+are unique per kind, ignoring case.
+
+| Code | | When |
+|---|---|---|
+| `zone_name`, `set_name`, `service_name`, `forward_name` | error | not a valid name, or one that looks like an address |
+| `zone_name_taken`, `set_name_taken`, `service_name_taken`, `forward_name_taken` | error | another object of the kind has that name |
+
+## Settings
+
+| Code | | When |
+|---|---|---|
+| `policy` | error | a policy that isn't `accept` or `drop` |
+| `settings_address` | error | an own prefix or antispoof entry that isn't an address or prefix, or is in the wrong list (an IPv6 prefix in the IPv4 list) |
+| `flowtable_device` | error | a flowtable device with a wildcard: name each device |
+| `flowtable_device_missing` | warning, *host* | a flowtable device doesn't exist; the flowtable is left out until it does |
+| `probation_range` | error | the probation isn't 10 to 3600 seconds |
+| `stats_no_external` | warning | traffic per device is on, but no zone is external: it counts what goes to and comes from external zones |
+| `stats_flowtable` | warning | traffic per device with the flowtable on: its fast path skips the forward chain, so only each connection's first packets are counted |
+
+## Zones
+
+| Code | | When |
+|---|---|---|
+| `zone_empty` | error | no interfaces |
+| `zone_interface` | error | not an interface name (up to 15 characters; a trailing `*` matches many) |
+| `zone_interface_taken` | error | the interface is in another zone already |
+| `zone_interface_missing` | warning, *host* | the interface doesn't exist here (yet); its rules match once it does |
+| `no_external_zone` | warning | no zone is external: antispoofing, MSS clamping, the DHCPv6 client rule and port forwards without public addresses need one, and the boot fallback lets nothing but loopback through without one ([operations.md](operations.md#if-the-ruleset-fails-to-load-at-boot)) |
+
+## Sets and services
+
+| Code | | When |
+|---|---|---|
+| `set_name_reserved` | error | a set name starting with `meter_`, `conn_`, `quota_`, `stats_` or `ban_`: Housecarl's own sets (rate limits, quotas, statistics, ban lists) use those |
+| `set_entry` | error | an entry that isn't an address, prefix (without host bits) or range |
+| `set_empty` | warning | a set (not dynamic, not from Jarl) with no entries: rules using it match nothing |
+| `set_dynamic_entries` | error | configured entries on a dynamic set: its entries come at run time |
+| `set_timeout` | error | a timeout on a set that isn't dynamic, or under a second |
+| `set_jarl_entries` | error | configured entries on a set from Jarl: its addresses are its names' ([jarl.md](jarl.md#sets-from-jarl)) |
+| `set_jarl_dynamic` | error | a set both dynamic and from Jarl |
+| `set_jarl_names` | error | a set from Jarl without names, with one that isn't a DNS name (a wildcard, say) or a bad view, or with a name twice |
+| `set_jarl_within` | error | a set from Jarl without `within`, or with an entry there that is neither a zone nor an address, network or range |
+| `set_jarl_anywhere` | warning | a set from Jarl `within 0.0.0.0/0` or `::/0`: any address may come in, so whoever changes the names in Jarl decides what it holds |
+| `service_empty` | error | a service with no entries |
+| `service_entry` | error | an entry that isn't `PROTO/PORTS` (`tcp`, `udp`, `sctp`; a port or range) or `icmp/TYPE`, `icmpv6/TYPE` with a type nftables knows |
+
+## Matches
+
+These apply to rules, source NAT and redirects.
+
+| Code | | When |
+|---|---|---|
+| `unknown_zone` | error | a zone that doesn't exist |
+| `match_in_zone` | error | in zones on the output chain (it has no incoming interface) |
+| `match_out_zone` | error | out zones on the input or early chain, or on a redirect |
+| `match_address` | error | an address that can't be read |
+| `unknown_set` | error | a name that is neither an address nor a set |
+| `unknown_service` | error | a service that is neither built in nor yours |
+| `family` | error | a family that isn't `ipv4` or `ipv6` |
+| `match_never` | error | the addresses, services and family can't all match the same packet (IPv4 sources and an `icmpv6` service) |
+
+## Rules
+
+| Code | | When |
+|---|---|---|
+| `rule_chain` | error | a chain that isn't `input`, `forward`, `output` or `early` |
+| `rule_action` | error | an action that isn't `accept`, `drop`, `reject` or `log` |
+| `raw_rule` | error | a raw rule over several lines, or with `;` |
+| `raw_conflict` | error | a raw rule with other fields set (zones, addresses, services, limits, log…): it is used as written, so they would be ignored |
+| `early_conntrack` | error | connection states, limits or a connection limit in the early chain, which runs before connection tracking |
+| `early_reject` | error | reject in the early chain: drop instead |
+| `ct_state`, `ct_status` | error | an unknown connection state or status |
+| `rule_day`, `rule_hours` | error | an unknown day, or hours that aren't `HH:MM`, or only one of them |
+| `rule_limit` | error | a rate that isn't a number per second, minute, hour or day |
+| `rule_conn_limit` | error | a connection limit under 1 |
+| `rule_limit_unit` | error | a rate's unit that isn't packets, bytes, kbytes or mbytes |
+| `rule_limit_bytes` | error | a byte rate nftables can't hold: over 4 GB a second, 293 MB a minute, 4.9 MB an hour or 208 KB a day (the message gives it per second) |
+| `rule_limit_key` | error | a limit both per source and per destination |
+| `rule_quota` | error | a quota under 1 byte |
+| `rule_quota_key` | error | a quota both per source and per destination |
+| `rule_quota_reset` | error | a reset that isn't daily, weekly or monthly; a day with another reset than monthly; a day outside 1-31 |
+| `rule_group` | error | a grouping prefix outside 1-32 (IPv4) or 1-128 (IPv6) |
+| `rule_traffic_accept` | error | a rule with a quota or a byte rate that accepts: it is checked before established connections are accepted, so it can only drop, reject or log |
+| `rule_every_packet_accept` | error | a rule checked against every packet (`every_packet`) that accepts: it comes before established connections are accepted, so it can only drop, reject or log |
+| `quota_restarts` | warning | a change makes a rule count its quota so that what it used can't carry over (a total to per address, per source to per destination, finer groups): it starts again. Only on the change that does it |
+| `flowtable_traffic` | warning | a forward rule checked against every packet (a quota, a byte rate, `every_packet`) while the flowtable is on: it only sees each connection's first packets |
+| `reject_with` | error | `tcp-reset` on a rule that matches more than TCP, or with an action other than reject |
+| `log_prefix` | error | a log prefix over 100 characters |
+
+## Port forwards
+
+| Code | | When |
+|---|---|---|
+| `forward_protocol` | error | no protocols, or one that isn't `tcp`, `udp` or `sctp` |
+| `forward_ports` | error | the public port isn't a port or range |
+| `forward_to_port` | error | the inside port isn't one port, or is given for a range of public ports |
+| `forward_target` | error | neither `to` nor `to6`, or `to` that isn't one IPv4 address, or `to6` one IPv6 address |
+| `forward_public` | error | no public addresses and no external zone, or an IPv6 public address |
+| `forward_overlap` | error | another forward has the same protocol, port and public address |
+| `forward_hairpin` | warning | hairpin without public addresses: inside clients' connections can't be recognised |
+
+## Source NAT and redirects
+
+| Code | | When |
+|---|---|---|
+| `snat_to` | error | `to` isn't one address, or of another family than the rule (give the rule the family) |
+| `snat_everything` | warning | no out zones and no sources: everything leaving the firewall is translated |
+| `redirect_service` | error | no service, or one without TCP, UDP or SCTP ports |
+| `redirect_to_port` | error | the local port isn't one port |
+
+## The machine
+
+These come from housecarld looking at the machine, not at the
+configuration: the dashboard and `hc status` show them.
+
+| Code | | When |
+|---|---|---|
+| `nftables_service` | warning, *host* | Debian's `nftables.service` is enabled: at boot it loads `/etc/nftables.conf`, and stopping or restarting it runs `nft flush ruleset`, deleting Housecarl's table too. Disable it with `systemctl disable nftables` (without `--now`) |
+| `conntrack_full` | warning, *host* | the connection tracking table is over 80% full; once it is full, the kernel drops new connections. Raise `net.netfilter.nf_conntrack_max`, or find what opens so many ([stats.md](stats.md#warnings)) |
+| `takeover_pending` | warning, *host* | the table you imported from is still in force beside Housecarl's, so a packet must get past both; `hc takeover TABLE` deletes it once Housecarl's rules are right |
+| `shadow_unprotected` | warning, *host* | shadow mode, and no other table has a filter chain on the input hook: Housecarl's ruleset blocks nothing, and nothing else protects the machine. Load your firewall, or leave shadow mode ([shadow.md](shadow.md)) |
+| `jarl_name` | warning, *host* | a set from Jarl has a name that gives no addresses, and why: no A, AAAA or CNAME record, a wildcard only, a CNAME out of Jarl's zones or in a loop, below a delegation, a zone Jarl doesn't hold the records of, or in more than one view without one named ([jarl.md](jarl.md#what-a-name-gives)) |
+| `jarl_outside` | warning, *host* | an address a name has in Jarl is left out of its set: it isn't within the set's zones and networks ([jarl.md](jarl.md#within)) |
+| `jarl_unreachable` | warning, *host* | reading Jarl failed (unreachable, the token refused…), for every name or one; the sets keep the addresses they had ([jarl.md](jarl.md#when-jarl-cant-be-reached)) |
+| `jarl_not_configured` | warning, *host* | sets from Jarl, but no `jarl.server` in `housecarl.yaml`: they keep the addresses they last had |
+| `shadow_raw_left_out` | warning, *host* | shadow mode leaves this raw rule out: it does more to packets than accept, drop or reject (NAT, setting a mark or field, a jump, a verdict map…). Its hits aren't counted, and the rules after it see what it would have taken ([shadow.md](shadow.md#what-changes-in-the-ruleset)) |
+
+In shadow mode `nftables_service` and `takeover_pending` aren't raised:
+another firewall beside Housecarl is the point there.
+
+## Ban lists
+
+| Code | | When |
+|---|---|---|
+| `banlist_name` | error | not a valid name (as for zones and sets) |
+| `banlist_name_taken` | error | two ban lists with one name |
+| `banlist_scope` | error | a scope other than `firewall`, `hosts` or `everything` |
+| `banlist_hosts` | error | scope `hosts` without hosts, or hosts with another scope |
+| `banlist_address` | error | a host or never entry that isn't an address, prefix or range |
+| `unknown_set` | error | a host or never entry naming a set that doesn't exist |
+| `banlist_time` | error | negative times (but `max_time` -1), or a default longer than the longest |
+| `banlist_entries` | error | `max_entries` below 0 or above 1000000 |
+| `banlist_group` | error | `group_v6` outside 16 to 128 |

go.mod

@@ -2,9 +2,31 @@ module housecarl
 
 go 1.27.1
 
-require github.com/spf13/cobra v1.10.2
+require (
+	github.com/aymanbagabas/go-udiff v0.4.1
+	github.com/go-acme/lego/v4 v4.35.2
+	github.com/miekg/dns v1.1.73
+	github.com/spf13/cobra v1.10.2
+	github.com/spf13/pflag v1.0.9
+	go.yaml.in/yaml/v3 v3.0.5
+	golang.org/x/crypto v0.57.0
+	golang.org/x/sys v0.48.0
+	golang.org/x/term v0.46.0
+	modernc.org/sqlite v1.60.1
+)
 
 require (
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
+	modernc.org/libc v1.77.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
 )

go.sum

@@ -1,10 +1,84 @@
+github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o=
+github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-acme/lego/v4 v4.35.2 h1:uVQg+KC/yj9R2g7Q9W5wDqhvQvxV5SMu5eqFVoN5xZU=
+github.com/go-acme/lego/v4 v4.35.2/go.mod h1:pX2jN5n8OphMGY1IaMjYm5DAEzguBaKRt8AvJAgJXpc=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
+github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
+golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
+golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
+modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
+modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
+modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
+modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/api/types.go

@@ -0,0 +1,606 @@
+// Package api holds the JSON shapes of Housecarl's REST API, shared by the
+// server and the Go client (and so the CLI).
+package api
+
+import (
+	"encoding/json"
+	"time"
+
+	"housecarl/internal/model"
+	"housecarl/internal/validate"
+	"housecarl/internal/version"
+)
+
+// Prefix starts every API path.
+const Prefix = "/api/v1"
+
+// ErrorBody is every error response.
+type ErrorBody struct {
+	Error ErrorInfo `json:"error"`
+}
+
+// ErrorInfo describes an error.
+type ErrorInfo struct {
+	Code     string             `json:"code"`
+	Message  string             `json:"message"`
+	Problems []validate.Problem `json:"problems,omitempty"`
+	// Result is the preview of a refused change.
+	Result json.RawMessage `json:"result,omitempty"`
+	// Op is the changeset operation (its seq) that couldn't be applied.
+	Op int `json:"op,omitempty"`
+}
+
+// Whoami describes the caller.
+type Whoami struct {
+	User        *model.User `json:"user"`
+	Admin       bool        `json:"admin"`
+	Method      string      `json:"method"`
+	Permissions []string    `json:"permissions"`
+	// TokenID is the token the caller authenticated with (0: none).
+	TokenID   int64         `json:"token_id,omitempty"`
+	CSRFToken string        `json:"csrf_token,omitempty"`
+	Build     version.Build `json:"build"`
+	// Mode is enforcing, or shadow (docs/shadow.md).
+	Mode string `json:"mode"`
+}
+
+// Modes: housecarld's ruleset decides, or (shadow mode, the
+// configuration file's shadow.enabled) it only shows what it would do,
+// beside another firewall.
+const (
+	ModeEnforcing = "enforcing"
+	ModeShadow    = "shadow"
+)
+
+// Login is the login request.
+type Login struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// PasswordChange changes the caller's password.
+type PasswordChange struct {
+	Current  string `json:"current"`
+	Password string `json:"password"`
+}
+
+// MutationResponse answers a configuration change.
+type MutationResponse struct {
+	Applied   bool               `json:"applied"`
+	DryRun    bool               `json:"dry_run,omitempty"`
+	HistoryID int64              `json:"history_id,omitempty"`
+	Changes   []model.Change     `json:"changes"`
+	Warnings  []validate.Problem `json:"warnings,omitempty"`
+	// Object is the created or changed object.
+	Object json.RawMessage `json:"object,omitempty"`
+	// ConfirmBy is set when the change is on probation: unless confirmed
+	// (POST /confirm) by then, it is rolled back.
+	ConfirmBy *time.Time `json:"confirm_by,omitempty"`
+	// ConfirmToken lets the client that made the change confirm it by
+	// itself (POST /confirm {"token"}): from the same address, over a new
+	// connection, which proves it can still reach the API. The answer
+	// closes its connection, so the next request opens a new one.
+	ConfirmToken string `json:"confirm_token,omitempty"`
+	// Diff is the ruleset's unified diff, for a dry run.
+	Diff string `json:"diff,omitempty"`
+	// Staged: the change was added to a changeset (Changeset), not
+	// applied.
+	Staged    bool             `json:"staged,omitempty"`
+	Changeset *model.Changeset `json:"changeset,omitempty"`
+}
+
+// ChangesetInput creates a changeset, or changes its description.
+type ChangesetInput struct {
+	Description string `json:"description"`
+}
+
+// ChangesetPreview is what applying a changeset would do: its changes,
+// the problems, nftables' check and the ruleset's diff.
+type ChangesetPreview struct {
+	// Valid: it can be applied as it is.
+	Valid    bool               `json:"valid"`
+	Changes  []model.Change     `json:"changes"`
+	Warnings []validate.Problem `json:"warnings,omitempty"`
+	// Error says why it can't be applied (Op: which operation, if one).
+	Error *ErrorInfo `json:"error,omitempty"`
+	Diff  string     `json:"diff,omitempty"`
+}
+
+// HostInterface is one of the server's network interfaces, with the zone
+// it is in.
+type HostInterface struct {
+	Name      string   `json:"name"`
+	Addresses []string `json:"addresses"`
+	Up        bool     `json:"up"`
+	Zone      string   `json:"zone,omitempty"`
+}
+
+// Counter is packets and bytes since the table was created.
+type Counter struct {
+	Packets uint64 `json:"packets"`
+	Bytes   uint64 `json:"bytes"`
+}
+
+// Counters are the hits of every object, by mark: "r12" (rule 12), "f3"
+// (port forward 3; "f3-6" its IPv6 pinhole, "f3-hairpin"), "s4", "d5",
+// "b-established" (a built-in), "p-input" (what reached a policy).
+type Counters struct {
+	Since    time.Time          `json:"since"`
+	Counters map[string]Counter `json:"counters"`
+}
+
+// Element is a dynamic set's entry.
+type Element struct {
+	Address string `json:"address"`
+	// Timeout in seconds: on input, how long it stays (0: the set's
+	// default); on output, how long it has left.
+	Timeout int `json:"timeout,omitempty"`
+}
+
+// Probation is the changes waiting to be confirmed.
+type Probation struct {
+	// Pending history entries, oldest first (empty: nothing pending).
+	Pending []int64 `json:"pending"`
+	// Takeovers is how many takeovers (deleted tables) are pending.
+	Takeovers int       `json:"takeovers,omitempty"`
+	Deadline  time.Time `json:"deadline,omitempty"`
+	// Seconds left before they are rolled back.
+	SecondsLeft int `json:"seconds_left,omitempty"`
+}
+
+// LogEntry is a packet a rule or a policy logged.
+type LogEntry struct {
+	Time     time.Time `json:"time"`
+	Prefix   string    `json:"prefix"`
+	In       string    `json:"in,omitempty"`
+	Out      string    `json:"out,omitempty"`
+	Family   string    `json:"family"`
+	Protocol string    `json:"protocol"`
+	Source   string    `json:"source"`
+	Dest     string    `json:"destination"`
+	SrcPort  int       `json:"source_port,omitempty"`
+	DestPort int       `json:"destination_port,omitempty"`
+	ICMPType string    `json:"icmp_type,omitempty"`
+	Length   int       `json:"length"`
+	TCPFlags string    `json:"tcp_flags,omitempty"`
+	RuleMark string    `json:"mark,omitempty"`
+	Verdict  string    `json:"verdict,omitempty"`
+	// Shadow: logged in shadow mode, where a drop or reject is what would
+	// have been done; the packet went on.
+	Shadow bool `json:"shadow,omitempty"`
+}
+
+// Status is the dashboard's overview.
+type Status struct {
+	Build    version.Build `json:"build"`
+	Started  time.Time     `json:"started"`
+	Database string        `json:"database,omitempty"`
+	// Mode is enforcing or shadow; ShadowPosition (shadow mode) after or
+	// before: where Housecarl's filter chains run beside the other
+	// firewall's.
+	Mode           string `json:"mode"`
+	ShadowPosition string `json:"shadow_position,omitempty"`
+	Applied        bool   `json:"applied"`
+	ApplyError     string `json:"apply_error,omitempty"`
+	// Unconfigured: the configuration is empty, and no ruleset was
+	// applied yet.
+	Unconfigured bool               `json:"unconfigured,omitempty"`
+	Problems     []validate.Problem `json:"problems"`
+	Probation    Probation          `json:"probation"`
+	Interfaces   []HostInterface    `json:"interfaces"`
+	Counts       map[string]int     `json:"counts"`
+	// Other tables in the ruleset (not Housecarl's), "inet main".
+	OtherTables []string `json:"other_tables"`
+}
+
+// MoveInput moves a rule (or NAT rule) before or after another, or to the
+// start or end of its chain; Chain moves a rule to another chain.
+type MoveInput struct {
+	Before int64  `json:"before,omitempty"`
+	After  int64  `json:"after,omitempty"`
+	To     string `json:"to,omitempty"` // start, end
+	Chain  string `json:"chain,omitempty"`
+}
+
+// ImportInput is an nftables ruleset to import: nft -j's JSON, and
+// optionally the ruleset file's text, whose defines name the sets.
+type ImportInput struct {
+	JSON json.RawMessage `json:"json"`
+	// Text is `nft -a list ruleset`: rules Housecarl can't express are
+	// kept from it as raw rules.
+	Text    string `json:"text,omitempty"`
+	Defines string `json:"defines,omitempty"`
+	// Table limits the import to one table ("inet main"; default: every
+	// table but Housecarl's).
+	Table string `json:"table,omitempty"`
+	// Replace replaces Housecarl's configuration; without it, the import
+	// is refused when there is one.
+	Replace bool `json:"replace,omitempty"`
+	// Zones names the zones by interface ({"enp2s0": "lan"}); interfaces
+	// given the same name share a zone.
+	Zones map[string]string `json:"zones,omitempty"`
+}
+
+// ImportResult reports an import.
+type ImportResult struct {
+	MutationResponse
+	// State is the imported configuration (also on a dry run).
+	State *model.State `json:"state"`
+	// Notes say how things were mapped; Raw lists what became raw rules.
+	Notes []string `json:"notes"`
+	Raw   []string `json:"raw"`
+}
+
+// TakeoverInput deletes the tables Housecarl replaces ("inet main").
+type TakeoverInput struct {
+	Tables []string `json:"tables"`
+}
+
+// TokenInput creates a token.
+type TokenInput struct {
+	Name string `json:"name"`
+	// Grants ("log:read", "sets:write:guest") or a role restrict the token; none: all of
+	// the user's permissions.
+	Grants    []string   `json:"grants,omitempty"`
+	Role      string     `json:"role,omitempty"`
+	ExpiresIn string     `json:"expires_in,omitempty"` // 90d, 720h
+	ExpiresAt *time.Time `json:"expires_at,omitempty"`
+}
+
+// TokenRotation gives a token a new secret. Grace keeps the old secret
+// working for a while (up to 7d). The token gets its lifetime again,
+// counted from now, unless ExpiresIn or NoExpiry says otherwise.
+type TokenRotation struct {
+	Grace     string `json:"grace,omitempty"`
+	ExpiresIn string `json:"expires_in,omitempty"`
+	NoExpiry  bool   `json:"no_expiry,omitempty"`
+}
+
+// Token is a token; Secret is only set when it is created.
+type Token struct {
+	*model.Token
+	GrantStrings []string `json:"grant_strings"`
+	Secret       string   `json:"secret,omitempty"`
+}
+
+// UserInput creates or changes a user.
+type UserInput struct {
+	Username *string   `json:"username,omitempty"`
+	Password *string   `json:"password,omitempty"`
+	Admin    *bool     `json:"admin,omitempty"`
+	Grants   *[]string `json:"grants,omitempty"`
+	Role     *string   `json:"role,omitempty"`
+	Disabled *bool     `json:"disabled,omitempty"`
+	Comment  *string   `json:"comment,omitempty"`
+}
+
+// User is a user with grants written out.
+type User struct {
+	*model.User
+	GrantStrings []string `json:"grant_strings"`
+}
+
+// CertStep is one step of the preflight check.
+type CertStep struct {
+	Name   string `json:"name"`
+	Status string `json:"status"` // ok, fail, warn, skip
+	Detail string `json:"detail"`
+	// Queries are the DNS questions asked, as dig commands.
+	Queries []string `json:"queries,omitempty"`
+}
+
+// CertCheck is the result of a preflight check: can Housecarl publish its
+// challenge through Jarl, and does the internet see it?
+type CertCheck struct {
+	Time   time.Time  `json:"time"`
+	Domain string     `json:"domain"`
+	OK     bool       `json:"ok"`
+	Steps  []CertStep `json:"steps"`
+}
+
+// CertStatus describes the certificate, for the API and the dashboard.
+type CertStatus struct {
+	Domain    string     `json:"domain"`
+	Directory string     `json:"directory"`
+	Staging   bool       `json:"staging"`
+	State     string     `json:"state"` // none, ok, working, error
+	NotBefore *time.Time `json:"not_before,omitempty"`
+	NotAfter  *time.Time `json:"not_after,omitempty"`
+	Issuer    string     `json:"issuer,omitempty"`
+	// IssuedBy is the directory the current certificate came from (it may
+	// differ from Directory until the next renewal).
+	IssuedBy    string     `json:"issued_by,omitempty"`
+	RenewAt     *time.Time `json:"renew_at,omitempty"`
+	LastError   string     `json:"last_error,omitempty"`
+	LastAttempt *time.Time `json:"last_attempt,omitempty"`
+	NextAttempt *time.Time `json:"next_attempt,omitempty"`
+	Check       *CertCheck `json:"check,omitempty"`
+}
+
+// CertificateResponse is GET /certificate.
+type CertificateResponse struct {
+	// Configured is false when tls.acme is off.
+	Configured bool `json:"configured"`
+	// Source is acme, file or none.
+	Source string      `json:"source"`
+	Status *CertStatus `json:"status,omitempty"`
+}
+
+// QuotaReset is POST /rules/{ref}/reset-quota's optional body. Without
+// one the whole quota starts again. Address starts one address of a
+// per-address quota again (any address in its group: 10.0.10.23, or an
+// IPv6 address for its /64); Give takes that many bytes off what was used
+// instead (the address's, or every address's), until the next reset.
+type QuotaReset struct {
+	Address string `json:"address,omitempty"`
+	Give    int64  `json:"give,omitempty"`
+}
+
+// QuotaInfo is a rule's quota and what it used.
+type QuotaInfo struct {
+	RuleID int64       `json:"rule_id"`
+	Rule   string      `json:"rule"`
+	Quota  model.Quota `json:"quota"`
+	// Used is a total quota's bytes; Addresses a per-address quota's,
+	// each ("192.168.1.20", or "2001:db8:1::/64").
+	Used      int64          `json:"used"`
+	Addresses []QuotaAddress `json:"addresses,omitempty"`
+	LastReset *time.Time     `json:"last_reset,omitempty"`
+	NextReset *time.Time     `json:"next_reset,omitempty"`
+}
+
+// QuotaAddress is one address's (or prefix's) use of a per-address quota.
+type QuotaAddress struct {
+	Address string `json:"address"`
+	Used    int64  `json:"used"`
+}
+
+// ConfirmInput confirms the changes on probation. With Token (a change's
+// confirm_token), it is the client that made the change confirming it by
+// itself; without, a person's confirmation.
+type ConfirmInput struct {
+	Token string `json:"token,omitempty"`
+}
+
+// Connection is a connection the kernel tracks (conntrack), as it was
+// first seen: Source:SourcePort → Destination:DestinationPort. NAT shows
+// as DNATTo (where it was really sent: a port forward) and SNATTo (the
+// source it went out with: masquerading).
+type Connection struct {
+	ID              uint32 `json:"id,omitempty"`
+	Family          string `json:"family"`
+	Protocol        string `json:"protocol"`
+	Source          string `json:"source"`
+	SourcePort      int    `json:"source_port,omitempty"`
+	Destination     string `json:"destination"`
+	DestinationPort int    `json:"destination_port,omitempty"`
+	ICMPID          int    `json:"icmp_id,omitempty"`
+	ICMPType        int    `json:"icmp_type,omitempty"`
+	ICMPCode        int    `json:"icmp_code,omitempty"`
+	DNATTo          string `json:"dnat_to,omitempty"`
+	DNATPort        int    `json:"dnat_port,omitempty"`
+	SNATTo          string `json:"snat_to,omitempty"`
+	// State is TCP's (established, time-wait, …); Assured: traffic went
+	// both ways.
+	State   string `json:"state,omitempty"`
+	Assured bool   `json:"assured,omitempty"`
+	// Timeout is the seconds left until it is forgotten unless more
+	// traffic comes.
+	Timeout int    `json:"timeout"`
+	Mark    uint32 `json:"mark,omitempty"`
+	// Packets and Bytes, both ways, when the kernel counts them
+	// (net.netfilter.nf_conntrack_acct).
+	Packets uint64 `json:"packets,omitempty"`
+	Bytes   uint64 `json:"bytes,omitempty"`
+}
+
+// NamesInput asks for the names of addresses (POST /names).
+type NamesInput struct {
+	Addresses []string `json:"addresses"`
+}
+
+// Names are the names found for the inside addresses asked about, by
+// address; those without one (or still being looked up) are left out.
+// Enabled is false when names are off (names.enabled).
+type Names struct {
+	Enabled bool              `json:"enabled"`
+	Names   map[string]string `json:"names"`
+}
+
+// CloseConnections closes connections: these, or every one Search finds
+// (as GET /connections?q= does).
+type CloseConnections struct {
+	Connections []Connection `json:"connections,omitempty"`
+	Search      string       `json:"search,omitempty"`
+}
+
+// TraceInput says what to trace: packets from or to Address (an address
+// or prefix), on Port (either way), of Protocol, for Seconds (0: 60).
+type TraceInput struct {
+	Address  string `json:"address,omitempty"`
+	Port     int    `json:"port,omitempty"`
+	Protocol string `json:"protocol,omitempty"`
+	Seconds  int    `json:"seconds,omitempty"`
+}
+
+// TraceStep is one rule or policy a traced packet met, in any table.
+type TraceStep struct {
+	Table string `json:"table"` // "inet housecarl", "ip nat"…
+	Chain string `json:"chain"`
+	Kind  string `json:"kind"` // rule or policy
+	// Rule is the rule as nftables writes it; Mark its Housecarl mark
+	// ("r12", "b-established"), if it is one of Housecarl's.
+	Rule    string `json:"rule,omitempty"`
+	Mark    string `json:"mark,omitempty"`
+	Verdict string `json:"verdict"` // accept, drop, continue, jump X…
+	// Label names Housecarl's object ("rule input #12 (ssh)").
+	Label string `json:"label,omitempty"`
+}
+
+// TracePacket is one traced packet: its headers as it arrived, every
+// step, and its fate.
+type TracePacket struct {
+	ID              string      `json:"id"`
+	Time            time.Time   `json:"time"`
+	In              string      `json:"in,omitempty"`
+	Out             string      `json:"out,omitempty"`
+	Protocol        string      `json:"protocol,omitempty"`
+	Source          string      `json:"source,omitempty"`
+	SourcePort      int         `json:"source_port,omitempty"`
+	Destination     string      `json:"destination,omitempty"`
+	DestinationPort int         `json:"destination_port,omitempty"`
+	TCPFlags        string      `json:"tcp_flags,omitempty"`
+	ICMPType        string      `json:"icmp_type,omitempty"`
+	Steps           []TraceStep `json:"steps"`
+	// Verdict is accepted or dropped; DecidedBy the step that decided.
+	Verdict   string    `json:"verdict"`
+	DecidedBy TraceStep `json:"decided_by"`
+	// WouldDrop, in shadow mode, is the step of Housecarl's where it
+	// would have dropped (or rejected) the packet, had it been enforcing.
+	WouldDrop *TraceStep `json:"would_drop,omitempty"`
+}
+
+// TraceStatus is the trace running (or the last one).
+type TraceStatus struct {
+	Running bool          `json:"running"`
+	Input   TraceInput    `json:"input"`
+	Started time.Time     `json:"started"`
+	Ends    time.Time     `json:"ends"`
+	Packets []TracePacket `json:"packets"`
+	Error   string        `json:"error,omitempty"`
+}
+
+// Matrix is what the rules do to new connections between zones:
+// zone → zone (the forward chain), zone → the firewall (input) and the
+// firewall → zone (output). "firewall" stands for the machine itself.
+type Matrix struct {
+	Zones []string     `json:"zones"`
+	Cells []MatrixCell `json:"cells"`
+}
+
+// MatrixCell is one pair: what happens to new connections from From to
+// To that no exception covers (Verdict, decided by DecidedBy), and the
+// exceptions: rules and built-ins that decide for some of them (some
+// addresses, services, times…), in the order they are checked.
+type MatrixCell struct {
+	From       string            `json:"from"`
+	To         string            `json:"to"`
+	Chain      string            `json:"chain"`
+	Verdict    string            `json:"verdict"` // allowed or blocked
+	DecidedBy  string            `json:"decided_by"`
+	RuleID     int64             `json:"rule_id,omitempty"` // the deciding rule
+	Exceptions []MatrixException `json:"exceptions"`
+	// Summary is the pair at a glance: allowed or blocked when every new
+	// connection gets the same, partly when some are let through and
+	// some not.
+	Summary string `json:"summary"`
+	// Allowed and Blocked count the exceptions that let part of the
+	// traffic through, and that block part of it.
+	Allowed int `json:"allowed"`
+	Blocked int `json:"blocked"`
+	// Open sums up what the allowing exceptions let through, shortest
+	// first in their order: "ssh", "dns", "5 port forwards".
+	Open []string `json:"open"`
+	// SameZone: from a zone to itself. Note says what of that traffic
+	// passes the firewall at all.
+	SameZone bool   `json:"same_zone,omitempty"`
+	Note     string `json:"note,omitempty"`
+	// Hits are the deciding rule's (or policy's) packets, all of its
+	// traffic since housecarld started (nil: no counter).
+	Hits *uint64 `json:"hits,omitempty"`
+	// Marks are the counters behind Hits.
+	Marks []string `json:"-"`
+}
+
+// MatrixException is a rule (or built-in) that decides for part of a
+// pair's traffic.
+type MatrixException struct {
+	RuleID int64  `json:"rule_id,omitempty"` // 0: a built-in, or port forwards
+	Chain  string `json:"chain"`
+	Label  string `json:"label"`
+	// Verdict is allowed, blocked, logged (and goes on), or unknown (a raw
+	// rule Housecarl can't read).
+	Verdict string `json:"verdict"`
+	// What it covers: "from trusted · ssh", "Monday 08:00–17:00".
+	What string `json:"what"`
+	// Short is What in a word or two, for the overview: its services,
+	// or else its first condition.
+	Short string `json:"short"`
+	// Hits are its packets, all of its traffic since housecarld started
+	// (a rule may apply to several pairs; nil: no counter).
+	Hits *uint64 `json:"hits,omitempty"`
+	// Marks are the counters behind Hits ("r12", "b-ping", "f3").
+	Marks []string `json:"-"`
+}
+
+// BanInput asks a ban list to ban an address (POST
+// /banlists/{list}/bans).
+type BanInput struct {
+	Address string `json:"address"`
+	// Protocol (tcp, udp) and Ports limit the ban; with neither, every
+	// port. Ports are numbers or service names, in a list or a string
+	// ("22", "ssh", "http,https"); "all", "0:65535" or "1-65535" mean
+	// every port. Ports without a protocol take the service's protocols,
+	// or for a number tcp and udp.
+	Protocol string          `json:"protocol,omitempty"`
+	Ports    json.RawMessage `json:"ports,omitempty"`
+	// Time is the ban's length in seconds: 0 the list's default, -1 its
+	// longest (or for ever). Longer than the list allows is cut down.
+	Time   int    `json:"time,omitempty"`
+	Source string `json:"source,omitempty"`
+	Reason string `json:"reason,omitempty"`
+}
+
+// BanResult is the answer to a ban: the bans made or renewed (one per
+// protocol and port), Existing for each already in force.
+type BanResult struct {
+	Bans []BanMade `json:"bans"`
+	// Closed counts the banned address's connections closed.
+	Closed int `json:"closed"`
+}
+
+// BanMade is a ban made, or renewed (Existing).
+type BanMade struct {
+	model.Ban
+	Existing bool `json:"existing,omitempty"`
+}
+
+// JarlSet is what a Jarl set holds and why (GET /sets/{ref}/jarl).
+type JarlSet struct {
+	Set string `json:"set"`
+	// Configured is whether housecarl.yaml names a Jarl server.
+	Configured bool `json:"configured"`
+	// Read is when housecarld last read Jarl's zones; Error why the last
+	// try failed (the set keeps the addresses it had).
+	Read  *time.Time `json:"read,omitempty"`
+	Error string     `json:"error,omitempty"`
+	// Entries are the addresses the set holds now.
+	Entries []string   `json:"entries"`
+	Names   []JarlName `json:"names"`
+	Within  []string   `json:"within"`
+}
+
+// JarlName is one of a Jarl set's names: what Jarl gave for it.
+type JarlName struct {
+	Name string `json:"name"`
+	View string `json:"view,omitempty"` // as configured ("": the only view that has it)
+	// Found is the view it was found in.
+	Found string `json:"found_view,omitempty"`
+	// Chain are the CNAME targets followed, in order.
+	Chain     []string      `json:"chain,omitempty"`
+	Addresses []JarlAddress `json:"addresses"`
+	// Problem is why Jarl's records give it no addresses.
+	Problem string `json:"problem,omitempty"`
+	// Checked is when Jarl last answered for it; Error why the last try
+	// failed (its addresses are those of Checked).
+	Checked *time.Time `json:"checked,omitempty"`
+	Error   string     `json:"error,omitempty"`
+}
+
+// JarlAddress is an address a name has in Jarl: Used if the set holds
+// it, else why not.
+type JarlAddress struct {
+	Address string `json:"address"`
+	Used    bool   `json:"used"`
+	Why     string `json:"why,omitempty"`
+}

internal/auth/auth.go

@@ -0,0 +1,343 @@
+// Package auth handles passwords, API tokens, sessions and permissions.
+package auth
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base32"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/crypto/argon2"
+
+	"housecarl/internal/model"
+)
+
+// Argon2id parameters (OWASP: m=19 MiB, t=2, p=1).
+const (
+	argonTime    = 2
+	argonMemory  = 19 * 1024
+	argonThreads = 1
+	argonKeyLen  = 32
+	saltLen      = 16
+)
+
+// MinPasswordLength is the shortest accepted password.
+const MinPasswordLength = 8
+
+// HashPassword returns an argon2id hash in PHC string format.
+func HashPassword(password string) (string, error) {
+	if len(password) < MinPasswordLength {
+		return "", fmt.Errorf("a password needs at least %d characters", MinPasswordLength)
+	}
+	salt := make([]byte, saltLen)
+	if _, err := rand.Read(salt); err != nil {
+		return "", err
+	}
+	key := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
+	return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemory, argonTime, argonThreads,
+		base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key)), nil
+}
+
+// dummyHash is checked when a user doesn't exist, so a login takes as
+// long whether or not the username is right.
+var dummyHash, _ = HashPassword("not-a-real-password")
+
+// CheckPassword reports whether password matches hash. An empty hash never
+// matches (in the same time).
+func CheckPassword(hash, password string) bool {
+	if hash == "" {
+		hash = dummyHash
+		password = "\x00" + password
+	}
+	parts := strings.Split(hash, "$")
+	if len(parts) != 6 || parts[1] != "argon2id" {
+		return false
+	}
+	var m, t uint32
+	var p uint8
+	for _, kv := range strings.Split(parts[3], ",") {
+		k, v, _ := strings.Cut(kv, "=")
+		n, err := strconv.ParseUint(v, 10, 32)
+		if err != nil {
+			return false
+		}
+		switch k {
+		case "m":
+			m = uint32(n)
+		case "t":
+			t = uint32(n)
+		case "p":
+			p = uint8(n)
+		}
+	}
+	salt, err1 := base64.RawStdEncoding.DecodeString(parts[4])
+	want, err2 := base64.RawStdEncoding.DecodeString(parts[5])
+	if err1 != nil || err2 != nil || m == 0 || t == 0 || p == 0 {
+		return false
+	}
+	got := argon2.IDKey([]byte(password), salt, t, m, p, uint32(len(want)))
+	return subtle.ConstantTimeCompare(got, want) == 1
+}
+
+var b32 = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567").WithPadding(base32.NoPadding)
+
+// RandomString returns a random string with nbytes of entropy.
+func RandomString(nbytes int) string {
+	b := make([]byte, nbytes)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return b32.EncodeToString(b)
+}
+
+// TokenPrefix starts every API token, so leaked ones are easy to spot.
+const TokenPrefix = "housecarl_"
+
+// NewToken makes a token: the whole string to give out, its public ID
+// (stored in clear) and the hash of its secret.
+func NewToken() (token, publicID, secretHash string) {
+	publicID = RandomString(10) // 16 characters
+	secret := RandomString(20)  // 32 characters
+	return TokenPrefix + publicID + "_" + secret, publicID, HashSecret(secret)
+}
+
+// RotateToken makes a new secret for a token's public ID.
+func RotateToken(publicID string) (token, secretHash string) {
+	secret := RandomString(20)
+	return TokenPrefix + publicID + "_" + secret, HashSecret(secret)
+}
+
+// ParseToken splits a token into its public ID and secret.
+func ParseToken(token string) (publicID, secret string, err error) {
+	rest, ok := strings.CutPrefix(strings.TrimSpace(token), TokenPrefix)
+	if !ok {
+		return "", "", errors.New("not a Housecarl token (they start with " + TokenPrefix + ")")
+	}
+	publicID, secret, ok = strings.Cut(rest, "_")
+	if !ok || len(publicID) != 16 || len(secret) != 32 {
+		return "", "", errors.New("malformed token")
+	}
+	return publicID, secret, nil
+}
+
+// HashSecret hashes a high-entropy secret (token secrets, session IDs).
+func HashSecret(secret string) string {
+	h := sha256.Sum256([]byte(secret))
+	return hex.EncodeToString(h[:])
+}
+
+// CheckSecret compares a secret with a stored hash in constant time.
+func CheckSecret(secret, hash string) bool {
+	return subtle.ConstantTimeCompare([]byte(HashSecret(secret)), []byte(hash)) == 1
+}
+
+// NewSessionID returns a session ID and its CSRF token.
+func NewSessionID() (id, csrf string) {
+	return RandomString(32), RandomString(32)
+}
+
+// Resources grants apply to.
+const (
+	ResConfig = "config" // the firewall configuration, its history and imports
+	ResSets   = "sets"   // dynamic sets' entries (a grant may name the sets)
+	ResBans   = "bans"   // ban lists' bans (a grant may name the lists)
+	ResLog    = "log"    // counters, the drop log and the status
+	ResAudit  = "audit"  // the audit log
+	ResAll    = "*"
+)
+
+// Resources lists them in display order.
+var Resources = []string{ResConfig, ResSets, ResBans, ResLog, ResAudit}
+
+// Access levels.
+const (
+	Read  = "read"
+	Write = "write"
+)
+
+// Roles are ready-made sets of grants.
+var Roles = map[string][]model.Grant{
+	"viewer":   {{Resource: ResConfig, Access: Read}, {Resource: ResSets, Access: Read}, {Resource: ResBans, Access: Read}, {Resource: ResLog, Access: Read}},
+	"operator": {{Resource: ResConfig, Access: Write}, {Resource: ResSets, Access: Write}, {Resource: ResBans, Access: Write}, {Resource: ResLog, Access: Read}, {Resource: ResAudit, Access: Read}},
+	"log":      {{Resource: ResLog, Access: Read}},
+}
+
+// RoleNames lists the roles.
+func RoleNames() []string {
+	var out []string
+	for n := range Roles {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// ParseGrant reads "resource:access" ("log:read", "*:write"), and for
+// sets and bans optionally the sets or ban lists it is limited to
+// ("sets:write:guest,portal", "bans:write:web").
+func ParseGrant(s string) (model.Grant, error) {
+	parts := strings.SplitN(strings.TrimSpace(s), ":", 3)
+	g := model.Grant{Resource: strings.ToLower(parts[0])}
+	if len(parts) < 2 {
+		return g, fmt.Errorf("grant %q: write it as resource:access, e.g. log:read, sets:write:SET1,SET2 or bans:write:LIST", s)
+	}
+	g.Access = strings.ToLower(parts[1])
+	if len(parts) == 3 {
+		for _, n := range strings.Split(parts[2], ",") {
+			if n = strings.TrimSpace(n); n != "" {
+				g.Sets = append(g.Sets, n)
+			}
+		}
+	}
+	return g, ValidateGrant(&g)
+}
+
+// ValidateGrant checks a grant.
+func ValidateGrant(g *model.Grant) error {
+	g.Resource = strings.ToLower(strings.TrimSpace(g.Resource))
+	g.Access = strings.ToLower(strings.TrimSpace(g.Access))
+	ok := g.Resource == ResAll
+	for _, r := range Resources {
+		ok = ok || g.Resource == r
+	}
+	if !ok {
+		return fmt.Errorf("unknown resource %q (use %s or *)", g.Resource, strings.Join(Resources, ", "))
+	}
+	if g.Access != Read && g.Access != Write {
+		return fmt.Errorf("access must be read or write, not %q", g.Access)
+	}
+	if len(g.Sets) > 0 && g.Resource != ResSets && g.Resource != ResBans {
+		return fmt.Errorf("only a sets or bans grant can name sets or ban lists (sets:write:%s)", strings.Join(g.Sets, ","))
+	}
+	return nil
+}
+
+// FormatGrant writes a grant as "resource:access" (and ":SET,…").
+func FormatGrant(g model.Grant) string {
+	if len(g.Sets) > 0 {
+		return g.Resource + ":" + g.Access + ":" + strings.Join(g.Sets, ",")
+	}
+	return g.Resource + ":" + g.Access
+}
+
+// Principal is who makes a request.
+type Principal struct {
+	User   *model.User
+	Token  *model.Token
+	Method string // session, token, socket
+	IP     string
+	Agent  string
+}
+
+// Name is the user's name.
+func (p *Principal) Name() string {
+	if p == nil || p.User == nil {
+		return ""
+	}
+	return p.User.Username
+}
+
+// IsAdmin reports whether the principal has full rights: an
+// administrator, not through a restricted token.
+func (p *Principal) IsAdmin() bool {
+	return p.User != nil && p.User.Admin && (p.Token == nil || p.Token.Grants == nil)
+}
+
+// Can reports whether the principal may read (write false) or change
+// (write true) a resource.
+func (p *Principal) Can(resource string, write bool) bool {
+	if p == nil || p.User == nil {
+		return false
+	}
+	allowed := p.User.Admin || covers(p.User.Grants, resource, write)
+	if p.Token != nil && p.Token.Grants != nil {
+		allowed = allowed && covers(p.Token.Grants, resource, write)
+	}
+	return allowed
+}
+
+func covers(gs []model.Grant, resource string, write bool) bool {
+	for _, g := range gs {
+		if (g.Resource == ResAll || g.Resource == resource) && (!write || g.Access == Write) {
+			return true
+		}
+	}
+	return false
+}
+
+// CanSet reports whether the principal may read or change the entries of
+// a dynamic set: a sets grant covers it if it names no sets, or names it.
+func (p *Principal) CanSet(name string, write bool) bool { return p.canNamed(ResSets, name, write) }
+
+// CanBan reports whether the principal may read or change the bans of a
+// ban list: a bans grant covers it if it names no lists, or names it.
+func (p *Principal) CanBan(list string, write bool) bool { return p.canNamed(ResBans, list, write) }
+
+func (p *Principal) canNamed(resource, name string, write bool) bool {
+	if p == nil || p.User == nil {
+		return false
+	}
+	allowed := p.User.Admin || coversNamed(p.User.Grants, resource, name, write)
+	if p.Token != nil && p.Token.Grants != nil {
+		allowed = allowed && coversNamed(p.Token.Grants, resource, name, write)
+	}
+	return allowed
+}
+
+func coversNamed(gs []model.Grant, resource, name string, write bool) bool {
+	for _, g := range gs {
+		if (g.Resource != ResAll && g.Resource != resource) || (write && g.Access != Write) {
+			continue
+		}
+		if len(g.Sets) == 0 {
+			return true
+		}
+		for _, s := range g.Sets {
+			if strings.EqualFold(s, name) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Permissions lists what the principal may do, for display.
+func (p *Principal) Permissions() []string {
+	var out []string
+	for _, r := range Resources {
+		switch {
+		case p.Can(r, true):
+			out = append(out, r+":write")
+		case p.Can(r, false):
+			out = append(out, r+":read")
+		}
+	}
+	if p.IsAdmin() {
+		out = append(out, "users:write")
+	}
+	return out
+}
+
+// Narrower reports whether every grant in a is covered by b.
+func Narrower(a, b []model.Grant) bool {
+	for _, g := range a {
+		if !covers(b, g.Resource, g.Access == Write) {
+			if g.Resource != ResAll {
+				return false
+			}
+			for _, r := range Resources {
+				if !covers(b, r, g.Access == Write) {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}

internal/auth/auth_test.go

@@ -0,0 +1,73 @@
+package auth
+
+import (
+	"testing"
+
+	"housecarl/internal/model"
+)
+
+func TestPasswords(t *testing.T) {
+	h, err := HashPassword("correct horse")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !CheckPassword(h, "correct horse") || CheckPassword(h, "wrong horse") || CheckPassword("", "") {
+		t.Error("password check")
+	}
+	if _, err := HashPassword("short"); err == nil {
+		t.Error("short password accepted")
+	}
+}
+
+func TestTokens(t *testing.T) {
+	tok, pid, hash := NewToken()
+	gotPID, secret, err := ParseToken(tok)
+	if err != nil || gotPID != pid || !CheckSecret(secret, hash) {
+		t.Fatalf("token %q: %v", tok, err)
+	}
+	if _, _, err := ParseToken("jarl_abc_def"); err == nil {
+		t.Error("a Jarl token was accepted")
+	}
+}
+
+func TestPermissions(t *testing.T) {
+	op := &model.User{Username: "op", Grants: Roles["operator"]}
+	p := &Principal{User: op}
+	if !p.Can(ResConfig, true) || !p.Can(ResAudit, false) || p.Can(ResAudit, true) || p.IsAdmin() {
+		t.Error("operator")
+	}
+	admin := &model.User{Username: "admin", Admin: true}
+	p = &Principal{User: admin, Token: &model.Token{Grants: []model.Grant{{Resource: ResLog, Access: Read}}}}
+	if !p.Can(ResLog, false) || p.Can(ResLog, true) || p.Can(ResConfig, false) || p.IsAdmin() {
+		t.Error("restricted admin token")
+	}
+	if !Narrower(Roles["log"], Roles["operator"]) || Narrower(Roles["operator"], Roles["viewer"]) {
+		t.Error("narrower")
+	}
+	if _, err := ParseGrant("log"); err == nil {
+		t.Error("grant without access accepted")
+	}
+	if g, err := ParseGrant("Config:Write"); err != nil || g.Resource != "config" {
+		t.Errorf("grant %v %v", g, err)
+	}
+}
+
+func TestSetGrants(t *testing.T) {
+	g, err := ParseGrant("sets:write:guest-signed-in, portal")
+	if err != nil || len(g.Sets) != 2 || FormatGrant(g) != "sets:write:guest-signed-in,portal" {
+		t.Fatalf("parse %+v %v", g, err)
+	}
+	if _, err := ParseGrant("config:write:guest"); err == nil {
+		t.Error("a config grant naming sets was accepted")
+	}
+	user := &model.User{Username: "portal", Grants: []model.Grant{g}}
+	p := &Principal{User: user}
+	if !p.CanSet("guest-signed-in", true) || !p.CanSet("PORTAL", false) || p.CanSet("trusted", false) || p.Can(ResConfig, false) {
+		t.Error("set-scoped user")
+	}
+	admin := &Principal{User: &model.User{Username: "admin", Admin: true},
+		Token: &model.Token{Grants: []model.Grant{{Resource: ResSets, Access: Write, Sets: []string{"guest-signed-in"}}}}}
+	if !admin.CanSet("guest-signed-in", true) || admin.CanSet("trusted", true) {
+		t.Error("an admin's set-scoped token")
+	}
+}

internal/certs/check.go

@@ -0,0 +1,379 @@
+package certs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"housecarl/internal/api"
+	"housecarl/internal/auth"
+	"housecarl/internal/jarl"
+)
+
+// Step and Check are api.CertStep and api.CertCheck.
+type (
+	Step  = api.CertStep
+	Check = api.CertCheck
+)
+
+// note records a query and its answer, replacing an earlier answer to
+// the same question (the checks ask again until the record shows up).
+func note(s *Step, q string) {
+	cmd, _, _ := strings.Cut(q, "  → ")
+	for i, old := range s.Queries {
+		if prev, _, _ := strings.Cut(old, "  → "); prev == cmd {
+			s.Queries[i] = q
+			return
+		}
+	}
+	s.Queries = append(s.Queries, q)
+}
+
+func add(c *Check, name, status, format string, args ...any) *Step {
+	c.Steps = append(c.Steps, Step{Name: name, Status: status, Detail: fmt.Sprintf(format, args...)})
+	return &c.Steps[len(c.Steps)-1]
+}
+
+// How long the check waits for the record to show up, and the name
+// servers' port (tests change them).
+var (
+	propagationWait = 90 * time.Second
+	pollEvery       = 2 * time.Second
+	nsPort          = "53"
+)
+
+// Preflight checks, without the CA, that a certificate can be obtained:
+// Jarl answers and knows the token, a zone holds the challenge name, the
+// token may write it, a test record can be written, the zone's name
+// servers and public resolvers serve it, and it can be removed again.
+func (m *Manager) Preflight(ctx context.Context) *Check {
+	c := &Check{Time: time.Now().UTC(), Domain: m.cfg.ACME.Domain}
+	defer func() {
+		c.OK = true
+		for _, s := range c.Steps {
+			if s.Status == "fail" {
+				c.OK = false
+			}
+		}
+		m.mu.Lock()
+		m.lastCheck = c
+		m.mu.Unlock()
+	}()
+	name := jarl.ChallengeName(m.cfg.ACME.Domain)
+	// A CNAME at the challenge name sends the challenge elsewhere (into a
+	// zone kept for challenges); the CA and lego follow it, so the check
+	// does too.
+	if target := m.cname(ctx, name); target != "" {
+		name = target
+	}
+	shown := strings.TrimSuffix(name, ".")
+
+	who, err := m.jarl.Whoami(ctx)
+	if err != nil {
+		add(c, "Jarl", "fail", "%s", explainJarl(err))
+		return c
+	}
+	add(c, "Jarl", "ok", "the token is %s's, on Jarl %s", who.User.Username, who.Build.Label)
+
+	all, err := m.jarl.Zones(ctx)
+	if err != nil {
+		add(c, "Zone", "fail", "listing the zones: %s", explainJarl(err))
+		return c
+	}
+	zones := jarl.ZonesFor(name, all, "")
+	if len(zones) == 0 {
+		add(c, "Zone", "fail", "no primary zone the token can see holds %s: add the zone to Jarl, or give the token's user access to it (zones=...)", shown)
+		return c
+	}
+	zone := strings.TrimSuffix(zones[0].Name, ".")
+	// Every view with the zone gets the record (linked zones through
+	// their link), so name them all.
+	var inViews []jarl.Zone
+	for _, z := range all {
+		if strings.EqualFold(z.Name, zones[0].Name) {
+			inViews = append(inViews, z)
+		}
+	}
+	add(c, "Zone", "ok", "%s%s", zone, viewsText(inViews))
+
+	value := "housecarl-check-" + auth.RandomString(8)
+	if err := m.jarl.SetTXT(ctx, name, value, zones, true, true, "Housecarl's certificate check"); err != nil {
+		add(c, "Permission", "fail", "%s", explainWrite(err, zone, shown))
+		return c
+	}
+	add(c, "Permission", "ok", "the token may write TXT records at %s", shown)
+
+	if err := m.jarl.SetTXT(ctx, name, value, zones, true, false, "Housecarl's certificate check"); err != nil {
+		add(c, "Write", "fail", "writing a test record: %s", explainWrite(err, zone, shown))
+		return c
+	}
+	add(c, "Write", "ok", "wrote TXT %q at %s; Jarl has it in named", value, shown)
+	defer func() {
+		clean, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+		defer cancel()
+		if err := m.jarl.SetTXT(clean, name, value, zones, false, false, "Housecarl's certificate check"); err != nil {
+			add(c, "Clean up", "fail", "removing the test record: %s (remove it in Jarl by hand)", explainJarl(err))
+			return
+		}
+		add(c, "Clean up", "ok", "removed the test record")
+	}()
+
+	authOK := true
+	if m.cfg.ACME.CheckPropagation {
+		authOK = m.checkNameServers(ctx, c, zone, name, value)
+	} else {
+		add(c, "Name servers", "skip", "tls.acme.check_propagation is off")
+	}
+	m.checkResolvers(ctx, c, name, value, authOK && m.cfg.ACME.CheckPropagation)
+	return c
+}
+
+func viewsText(zs []jarl.Zone) string {
+	var views []string
+	for _, z := range zs {
+		if z.View != "" {
+			views = append(views, z.View)
+		}
+	}
+	switch len(views) {
+	case 0:
+		return ""
+	case 1:
+		return " in view " + views[0]
+	}
+	return " in views " + strings.Join(views, ", ")
+}
+
+func explainJarl(err error) string {
+	var je *jarl.Error
+	if errors.As(err, &je) && je.Status == 401 {
+		return "Jarl refused the token (" + je.Message + "): check jarl.token or jarl.token_file"
+	}
+	return err.Error()
+}
+
+func explainWrite(err error, zone, name string) string {
+	var je *jarl.Error
+	if errors.As(err, &je) && je.Status == 403 {
+		return fmt.Sprintf("the token may not write the challenge record (%s); give it the grant: records:write zones=%s names=%s types=TXT", je.Message, zone, name)
+	}
+	return explainJarl(err)
+}
+
+// checkNameServers asks each of the zone's public name servers directly
+// (as the CA's resolvers will) until all serve the value.
+func (m *Manager) checkNameServers(ctx context.Context, c *Check, zone, name, value string) bool {
+	st := add(c, "Name servers", "fail", "")
+	ns, q, err := m.lookupNS(ctx, zone)
+	st.Queries = append(st.Queries, q...)
+	if err != nil || len(ns) == 0 {
+		st.Detail = fmt.Sprintf("the public resolvers know no name servers for %s, so the CA can't see it: is the zone delegated to your name servers? (%v)", zone, err)
+		return false
+	}
+	type server struct {
+		host  string
+		addrs []string
+	}
+	var servers []server
+	for _, h := range ns {
+		addrs, q := m.lookupAddrs(ctx, h)
+		st.Queries = append(st.Queries, q...)
+		if len(addrs) == 0 {
+			st.Detail = fmt.Sprintf("name server %s has no address the public resolvers know", strings.TrimSuffix(h, "."))
+			return false
+		}
+		servers = append(servers, server{h, addrs})
+	}
+	// A server passes when any of its addresses serves the value (this
+	// machine may not reach its IPv6 address, for one).
+	deadline := time.Now().Add(propagationWait)
+	missing := servers
+	for {
+		var still []server
+		for _, s := range missing {
+			found := false
+			for _, a := range s.addrs {
+				ok, q := m.hasTXT(ctx, net.JoinHostPort(a, nsPort), name, value, false)
+				note(st, q)
+				if ok {
+					found = true
+					break
+				}
+			}
+			if !found {
+				still = append(still, s)
+			}
+		}
+		missing = still
+		if len(missing) == 0 {
+			st.Status, st.Detail = "ok", fmt.Sprintf("all %d name servers of %s serve it", len(servers), zone)
+			return true
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			var names []string
+			for _, s := range missing {
+				names = append(names, strings.TrimSuffix(s.host, ".")+" ("+strings.Join(s.addrs, ", ")+")")
+			}
+			st.Detail = fmt.Sprintf("after %s, %s still don't serve it: a secondary that doesn't get the zone's updates (check NOTIFY and transfers), or an address that doesn't reach your name server", propagationWait, strings.Join(names, ", "))
+			return false
+		}
+		sleep(ctx, pollEvery)
+	}
+}
+
+// checkResolvers asks the public resolvers until each returns the value.
+// When the name servers already serve it, a resolver that doesn't is only
+// a warning: it may be holding an earlier answer.
+func (m *Manager) checkResolvers(ctx context.Context, c *Check, name, value string, soft bool) {
+	st := add(c, "Public resolvers", "fail", "")
+	if len(m.cfg.ACME.Resolvers) == 0 {
+		st.Status, st.Detail = "skip", "no tls.acme.resolvers"
+		return
+	}
+	deadline := time.Now().Add(propagationWait)
+	missing := append([]string(nil), m.cfg.ACME.Resolvers...)
+	for {
+		var still []string
+		for _, r := range missing {
+			ok, q := m.hasTXT(ctx, r, name, value, true)
+			note(st, q)
+			if !ok {
+				still = append(still, r)
+			}
+		}
+		missing = still
+		if len(missing) == 0 {
+			st.Status, st.Detail = "ok", "every resolver sees it"
+			return
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			st.Detail = fmt.Sprintf("after %s, %s don't see it", propagationWait, strings.Join(missing, ", "))
+			if soft {
+				st.Status = "warn"
+				st.Detail += "; the name servers do, so they are probably holding an earlier answer"
+			}
+			return
+		}
+		sleep(ctx, pollEvery)
+	}
+}
+
+func sleep(ctx context.Context, d time.Duration) {
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// exchange asks server (HOST:PORT) one question.
+func exchange(ctx context.Context, server, name string, qtype uint16, recurse bool) (*dns.Msg, string, error) {
+	if _, _, err := net.SplitHostPort(server); err != nil {
+		server = net.JoinHostPort(server, "53")
+	}
+	m := new(dns.Msg)
+	m.SetQuestion(dns.Fqdn(name), qtype)
+	m.RecursionDesired = recurse
+	host, port, _ := net.SplitHostPort(server)
+	q := fmt.Sprintf("dig @%s -p %s %s %s", host, port, dns.TypeToString[qtype], strings.TrimSuffix(name, "."))
+	if !recurse {
+		q += " +norec"
+	}
+	c, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	cl := &dns.Client{Timeout: 5 * time.Second}
+	r, _, err := cl.ExchangeContext(c, m, server)
+	if err == nil && r.Truncated {
+		cl.Net = "tcp"
+		r, _, err = cl.ExchangeContext(c, m, server)
+	}
+	return r, q, err
+}
+
+func (m *Manager) hasTXT(ctx context.Context, server, name, value string, recurse bool) (bool, string) {
+	r, q, err := exchange(ctx, server, name, dns.TypeTXT, recurse)
+	if err != nil {
+		return false, q + "  → " + err.Error()
+	}
+	for _, rr := range r.Answer {
+		if t, ok := rr.(*dns.TXT); ok && strings.Join(t.Txt, "") == value {
+			return true, q + "  → found"
+		}
+	}
+	return false, q + "  → " + dns.RcodeToString[r.Rcode] + ", not there (yet)"
+}
+
+// cname returns the target of a CNAME at name ("" when there is none).
+func (m *Manager) cname(ctx context.Context, name string) string {
+	for _, res := range m.cfg.ACME.Resolvers {
+		r, _, err := exchange(ctx, res, name, dns.TypeCNAME, true)
+		if err != nil {
+			continue
+		}
+		for _, rr := range r.Answer {
+			if c, ok := rr.(*dns.CNAME); ok {
+				return strings.ToLower(c.Target)
+			}
+		}
+		return ""
+	}
+	return ""
+}
+
+// lookupNS asks the resolvers for a zone's name servers.
+func (m *Manager) lookupNS(ctx context.Context, zone string) ([]string, []string, error) {
+	var queries []string
+	var last error = errors.New("no resolvers")
+	for _, res := range m.cfg.ACME.Resolvers {
+		r, q, err := exchange(ctx, res, zone, dns.TypeNS, true)
+		queries = append(queries, q)
+		if err != nil {
+			last = err
+			continue
+		}
+		var out []string
+		for _, rr := range r.Answer {
+			if ns, ok := rr.(*dns.NS); ok {
+				out = append(out, strings.ToLower(ns.Ns))
+			}
+		}
+		if len(out) > 0 {
+			sort.Strings(out)
+			return out, queries, nil
+		}
+		last = fmt.Errorf("%s answered %s", res, dns.RcodeToString[r.Rcode])
+	}
+	return nil, queries, last
+}
+
+// lookupAddrs asks the resolvers for a host's addresses.
+func (m *Manager) lookupAddrs(ctx context.Context, host string) ([]string, []string) {
+	var queries []string
+	for _, res := range m.cfg.ACME.Resolvers {
+		var out []string
+		for _, t := range []uint16{dns.TypeA, dns.TypeAAAA} {
+			r, q, err := exchange(ctx, res, host, t, true)
+			queries = append(queries, q)
+			if err != nil {
+				continue
+			}
+			for _, rr := range r.Answer {
+				switch a := rr.(type) {
+				case *dns.A:
+					out = append(out, a.A.String())
+				case *dns.AAAA:
+					out = append(out, a.AAAA.String())
+				}
+			}
+		}
+		if len(out) > 0 {
+			return out, queries
+		}
+	}
+	return nil, queries
+}

internal/certs/check_test.go

@@ -0,0 +1,266 @@
+package certs
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"housecarl/internal/config"
+	"housecarl/internal/jarl"
+)
+
+// fakeJarl is Jarl's API as far as Housecarl uses it, and the DNS server
+// that serves what it writes (both the "public resolver" and the zone's
+// name server).
+type fakeJarl struct {
+	mu        sync.Mutex
+	txt       map[string][]string // name -> values
+	canWrite  bool
+	token     string
+	served    bool // the DNS server serves what is written
+	zones     []jarl.Zone
+	writes    int
+	dryRuns   int
+	nsHasIPv6 bool
+}
+
+func (f *fakeJarl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Header.Get("Authorization") != "Bearer "+f.token {
+		w.WriteHeader(401)
+		w.Write([]byte(`{"error":{"code":"unauthenticated","message":"invalid API token: unknown token"}}`))
+		return
+	}
+	switch r.URL.Path {
+	case "/api/v1/auth/whoami":
+		w.Write([]byte(`{"user":{"username":"housecarl"},"build":{"label":"v0.6.1"}}`))
+	case "/api/v1/zones":
+		json.NewEncoder(w).Encode(f.zones)
+	case "/api/v1/apply":
+		var b struct {
+			Ops []struct {
+				Op   string         `json:"op"`
+				Data map[string]any `json:"data"`
+			} `json:"ops"`
+			DryRun bool `json:"dry_run"`
+		}
+		json.NewDecoder(r.Body).Decode(&b)
+		if !f.canWrite {
+			w.WriteHeader(403)
+			w.Write([]byte(`{"error":{"code":"forbidden","message":"permission denied: this needs records:write"}}`))
+			return
+		}
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if b.DryRun {
+			f.dryRuns++
+			w.Write([]byte(`{}`))
+			return
+		}
+		f.writes++
+		for _, op := range b.Ops {
+			name, v := op.Data["name"].(string), op.Data["data"].(string)
+			var keep []string
+			for _, x := range f.txt[name] {
+				if x != v {
+					keep = append(keep, x)
+				}
+			}
+			if op.Op == "record.create" {
+				keep = append(keep, v)
+			}
+			f.txt[name] = keep
+		}
+		w.Write([]byte(`{}`))
+	default:
+		w.WriteHeader(404)
+	}
+}
+
+func (f *fakeJarl) dns(w dns.ResponseWriter, r *dns.Msg) {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	q := r.Question[0]
+	hdr := dns.RR_Header{Name: q.Name, Class: dns.ClassINET, Ttl: 60}
+	switch q.Qtype {
+	case dns.TypeNS:
+		if q.Name == "adbraun.test." {
+			hdr.Rrtype = dns.TypeNS
+			m.Answer = append(m.Answer, &dns.NS{Hdr: hdr, Ns: "ns1.adbraun.test."})
+		} else {
+			m.Rcode = dns.RcodeNameError
+		}
+	case dns.TypeA:
+		hdr.Rrtype = dns.TypeA
+		m.Answer = append(m.Answer, &dns.A{Hdr: hdr, A: net.ParseIP("127.0.0.1")})
+	case dns.TypeAAAA:
+		// An IPv6 address nothing answers on (the check must not fail
+		// because of it).
+		if f.nsHasIPv6 {
+			hdr.Rrtype = dns.TypeAAAA
+			m.Answer = append(m.Answer, &dns.AAAA{Hdr: hdr, AAAA: net.ParseIP("100::1")})
+		}
+	case dns.TypeCNAME:
+		if q.Name == "_acme-challenge.dhcp.other.test." {
+			hdr.Rrtype = dns.TypeCNAME
+			m.Answer = append(m.Answer, &dns.CNAME{Hdr: hdr, Target: "dhcp.acme.adbraun.test."})
+		}
+	case dns.TypeTXT:
+		f.mu.Lock()
+		if f.served {
+			for _, v := range f.txt[q.Name] {
+				hdr.Rrtype = dns.TypeTXT
+				m.Answer = append(m.Answer, &dns.TXT{Hdr: hdr, Txt: []string{v}})
+			}
+		}
+		f.mu.Unlock()
+	}
+	w.WriteMsg(m)
+}
+
+func setup(t *testing.T, f *fakeJarl, edit func(a *config.ACME)) (*Manager, string) {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(f.dns)}
+	go srv.ActivateAndServe()
+	t.Cleanup(func() { srv.Shutdown() })
+	hs := httptest.NewServer(f)
+	t.Cleanup(hs.Close)
+	addr := pc.LocalAddr().String()
+	_, port, _ := net.SplitHostPort(addr)
+	oldPort, oldWait, oldPoll := nsPort, propagationWait, pollEvery
+	nsPort, propagationWait, pollEvery = port, 2*time.Second, 100*time.Millisecond
+	t.Cleanup(func() { nsPort, propagationWait, pollEvery = oldPort, oldWait, oldPoll })
+	a := config.ACME{Domain: "dhcp.adbraun.test", Resolvers: []string{addr}, CheckPropagation: true}
+	if edit != nil {
+		edit(&a)
+	}
+	jc, _ := jarl.New(hs.URL, "housecarl_good", "", false)
+	m := &Manager{cfg: Config{ACME: a}, jarl: jc, now: time.Now}
+	return m, addr
+}
+
+func steps(c *Check) string {
+	var out []string
+	for _, s := range c.Steps {
+		out = append(out, s.Name+"="+s.Status)
+	}
+	return strings.Join(out, " ")
+}
+
+func newFake() *fakeJarl {
+	return &fakeJarl{txt: map[string][]string{}, canWrite: true, token: "housecarl_good", served: true,
+		zones: []jarl.Zone{{Name: "adbraun.test.", View: "internal", LinkGroup: 7}, {Name: "adbraun.test.", View: "external", LinkGroup: 7}, {Name: "other.test.", View: "internal"}}}
+}
+
+func TestPreflightOK(t *testing.T) {
+	f := newFake()
+	f.nsHasIPv6 = true
+	m, _ := setup(t, f, nil)
+	c := m.Preflight(context.Background())
+	want := "Jarl=ok Zone=ok Permission=ok Write=ok Name servers=ok Public resolvers=ok Clean up=ok"
+	if !c.OK || steps(c) != want {
+		t.Fatalf("got %s\n%+v", steps(c), c.Steps)
+	}
+	if !strings.Contains(c.Steps[1].Detail, "views internal, external") {
+		t.Errorf("zone step: %s", c.Steps[1].Detail)
+	}
+	// Linked zones are written once; the test record is gone afterwards.
+	if f.dryRuns != 1 || f.writes != 2 || len(f.txt["_acme-challenge.dhcp.adbraun.test."]) != 0 {
+		t.Errorf("dry runs %d writes %d records %v", f.dryRuns, f.writes, f.txt)
+	}
+}
+
+func TestPreflightFailures(t *testing.T) {
+	cases := []struct {
+		name string
+		edit func(f *fakeJarl, a *config.ACME)
+		want string
+		text string
+	}{
+		{"bad token", func(f *fakeJarl, a *config.ACME) { f.token = "other" }, "Jarl=fail", "check jarl.token"},
+		{"no zone", func(f *fakeJarl, a *config.ACME) { a.Domain = "dhcp.example.net" }, "Jarl=ok Zone=fail", "no primary zone the token can see"},
+		{"no permission", func(f *fakeJarl, a *config.ACME) { f.canWrite = false }, "Jarl=ok Zone=ok Permission=fail",
+			"records:write zones=adbraun.test names=_acme-challenge.dhcp.adbraun.test types=TXT"},
+		{"not served", func(f *fakeJarl, a *config.ACME) { f.served = false },
+			"Jarl=ok Zone=ok Permission=ok Write=ok Name servers=fail Public resolvers=fail Clean up=ok", "still don't serve it"},
+		{"not delegated", func(f *fakeJarl, a *config.ACME) { f.zones[0].Name, f.zones[1].Name = "test.", "test." },
+			"Jarl=ok Zone=ok Permission=ok Write=ok Name servers=fail Public resolvers=ok Clean up=ok", "know no name servers for test"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFake()
+			var am *config.ACME
+			m, _ := setup(t, f, func(a *config.ACME) { am = a })
+			tc.edit(f, am)
+			m.cfg.ACME = *am
+			c := m.Preflight(context.Background())
+			if c.OK || steps(c) != tc.want {
+				t.Fatalf("got %s, want %s\n%+v", steps(c), tc.want, c.Steps)
+			}
+			all := ""
+			for _, s := range c.Steps {
+				all += s.Detail + "\n"
+			}
+			if !strings.Contains(all, tc.text) {
+				t.Errorf("details lack %q:\n%s", tc.text, all)
+			}
+		})
+	}
+}
+
+func TestResolversOnlyWarnWhenNameServersServe(t *testing.T) {
+	f := newFake()
+	m, _ := setup(t, f, nil)
+	// The "public resolver" is another server that never sees the record.
+	pc, _ := net.ListenPacket("udp", "127.0.0.1:0")
+	srv := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
+		mm := new(dns.Msg)
+		mm.SetReply(r)
+		if r.Question[0].Qtype == dns.TypeTXT {
+			mm.Rcode = dns.RcodeNameError
+		} else {
+			f.dns(w, r)
+			return
+		}
+		w.WriteMsg(mm)
+	})}
+	go srv.ActivateAndServe()
+	defer srv.Shutdown()
+	m.cfg.ACME.Resolvers = []string{pc.LocalAddr().String()}
+	c := m.Preflight(context.Background())
+	if !c.OK || !strings.Contains(steps(c), "Public resolvers=warn") {
+		t.Fatalf("got %s %+v", steps(c), c.Steps)
+	}
+}
+
+func TestPreflightFollowsCNAME(t *testing.T) {
+	f := newFake()
+	m, _ := setup(t, f, func(a *config.ACME) { a.Domain = "dhcp.other.test" })
+	c := m.Preflight(context.Background())
+	if !c.OK || !strings.Contains(c.Steps[2].Detail, "dhcp.acme.adbraun.test") {
+		t.Fatalf("got %s %+v", steps(c), c.Steps)
+	}
+}
+
+func TestZonesFor(t *testing.T) {
+	zs := []jarl.Zone{{Name: "lol.", View: "a"}, {Name: "adbraun.lol.", View: "internal", LinkGroup: 3}, {Name: "adbraun.lol.", View: "external", LinkGroup: 3},
+		{Name: "adbraun.lol.", View: "guest"}, {Name: "notadbraun.lol.", View: "internal"}}
+	got := jarl.ZonesFor("_acme-challenge.dhcp.adbraun.lol.", zs, "")
+	if len(got) != 2 || got[0].View != "internal" || got[1].View != "guest" {
+		t.Errorf("got %+v", got)
+	}
+	if n := jarl.ChallengeName("*.Dhcp.adbraun.lol."); n != "_acme-challenge.dhcp.adbraun.lol." {
+		t.Errorf("challenge name %q", n)
+	}
+}

internal/certs/manager.go

@@ -0,0 +1,484 @@
+// Package certs obtains and renews the web UI's certificate from an ACME CA
+// (Let's Encrypt), with Jarl publishing the DNS-01 challenge
+// (docs/certificates.md). Before every order a preflight check proves the
+// challenge can be published and seen, so a broken setup never spends the
+// CA's rate limits.
+package certs
+
+import (
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-acme/lego/v4/certificate"
+	"github.com/go-acme/lego/v4/challenge/dns01"
+	"github.com/go-acme/lego/v4/lego"
+	legolog "github.com/go-acme/lego/v4/log"
+	"github.com/go-acme/lego/v4/registration"
+
+	"housecarl/internal/api"
+	"housecarl/internal/config"
+	"housecarl/internal/jarl"
+)
+
+// Store keeps the account and certificate (the database's meta table).
+type Store interface {
+	GetMeta(ctx context.Context, key string) (string, error)
+	SetMeta(ctx context.Context, key, value string) error
+}
+
+const (
+	metaAccount = "acme.account"
+	metaCert    = "acme.certificate"
+)
+
+// Config configures the manager.
+type Config struct {
+	ACME    config.ACME
+	Version string
+}
+
+// Status is api.CertStatus.
+type Status = api.CertStatus
+
+// Manager keeps the web UI's certificate.
+type Manager struct {
+	cfg   Config
+	jarl  *jarl.Client
+	store Store
+	log   *slog.Logger
+	now   func() time.Time
+
+	mu          sync.Mutex
+	cert        *tls.Certificate
+	leaf        *x509.Certificate
+	issuedBy    string
+	working     bool
+	lastErr     string
+	lastAttempt time.Time
+	nextAttempt time.Time
+	lastCheck   *Check
+	failures    int
+	force       bool // renew now, even if not due (Kick)
+
+	lego sync.Mutex // one ACME order at a time
+	kick chan struct{}
+}
+
+// New makes a manager and loads the stored certificate.
+func New(ctx context.Context, cfg Config, j *jarl.Client, st Store, log *slog.Logger) *Manager {
+	if log == nil {
+		log = slog.Default()
+	}
+	// lego logs each step of an order; send it to the journal with the rest.
+	legolog.Logger = slog.NewLogLogger(log.With("component", "acme").Handler(), slog.LevelInfo)
+	m := &Manager{cfg: cfg, jarl: j, store: st, log: log, now: time.Now, kick: make(chan struct{}, 1)}
+	m.load(ctx)
+	return m
+}
+
+type stored struct {
+	Domain    string    `json:"domain"`
+	Directory string    `json:"directory"`
+	CertPEM   string    `json:"cert_pem"`
+	KeyPEM    string    `json:"key_pem"`
+	Obtained  time.Time `json:"obtained"`
+}
+
+func (m *Manager) load(ctx context.Context) {
+	raw, err := m.store.GetMeta(ctx, metaCert)
+	if err != nil || raw == "" {
+		return
+	}
+	var s stored
+	if json.Unmarshal([]byte(raw), &s) != nil {
+		return
+	}
+	if err := m.use(s); err != nil {
+		m.log.Warn("the stored certificate can't be used; getting a new one", "err", err)
+	}
+}
+
+// use makes a stored certificate the one served.
+func (m *Manager) use(s stored) error {
+	c, err := tls.X509KeyPair([]byte(s.CertPEM), []byte(s.KeyPEM))
+	if err != nil {
+		return err
+	}
+	leaf, err := x509.ParseCertificate(c.Certificate[0])
+	if err != nil {
+		return err
+	}
+	c.Leaf = leaf
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	// A certificate for another name isn't served.
+	if !strings.EqualFold(s.Domain, m.cfg.ACME.Domain) {
+		m.cert, m.leaf = nil, nil
+		return fmt.Errorf("it is for %s, not %s", s.Domain, m.cfg.ACME.Domain)
+	}
+	m.cert, m.leaf, m.issuedBy = &c, leaf, s.Directory
+	return nil
+}
+
+// Certificate is the certificate to serve (nil when there is none yet).
+func (m *Manager) Certificate() *tls.Certificate {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.cert == nil || m.now().After(m.leaf.NotAfter) {
+		return nil
+	}
+	return m.cert
+}
+
+// Status describes the certificate.
+func (m *Manager) Status() Status {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	st := Status{Domain: m.cfg.ACME.Domain, Directory: m.cfg.ACME.DirectoryURL(), Staging: m.cfg.ACME.Staging(), State: "none",
+		IssuedBy: m.issuedBy, LastError: m.lastErr, Check: m.lastCheck}
+	if m.leaf != nil {
+		nb, na := m.leaf.NotBefore, m.leaf.NotAfter
+		ra := m.renewAt()
+		st.NotBefore, st.NotAfter, st.RenewAt, st.Issuer, st.State = &nb, &na, &ra, m.leaf.Issuer.CommonName, "ok"
+	}
+	if m.lastErr != "" {
+		st.State = "error"
+	}
+	if m.working {
+		st.State = "working"
+	}
+	if !m.lastAttempt.IsZero() {
+		t := m.lastAttempt
+		st.LastAttempt = &t
+	}
+	if !m.nextAttempt.IsZero() {
+		t := m.nextAttempt
+		st.NextAttempt = &t
+	}
+	return st
+}
+
+// renewAt is when the certificate is renewed: a third of its lifetime
+// before it ends (30 days for Let's Encrypt's 90), m.mu held.
+func (m *Manager) renewAt() time.Time {
+	life := m.leaf.NotAfter.Sub(m.leaf.NotBefore)
+	return m.leaf.NotAfter.Add(-life / 3).UTC().Truncate(time.Second)
+}
+
+// due reports whether a new certificate is needed: none yet, another
+// directory configured, or renewal time.
+func (m *Manager) due() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.force || m.leaf == nil || m.issuedBy != m.cfg.ACME.DirectoryURL() || !m.now().Before(m.renewAt())
+}
+
+// Run keeps the certificate: it gets one when needed, checks twice a day
+// whether it is due, and retries failures with a growing wait.
+func (m *Manager) Run(ctx context.Context) {
+	timer := time.NewTimer(5 * time.Second)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-m.kick:
+		case <-timer.C:
+		}
+		wait := 12 * time.Hour
+		if m.due() {
+			if err := m.Renew(ctx); err != nil {
+				m.mu.Lock()
+				m.failures++
+				// 1h, 3h, 6h, then 12h.
+				waits := []time.Duration{time.Hour, 3 * time.Hour, 6 * time.Hour, 12 * time.Hour}
+				wait = waits[min(m.failures, len(waits))-1]
+				m.mu.Unlock()
+				m.log.Warn("getting the web UI's certificate failed; trying again later", "domain", m.cfg.ACME.Domain, "err", err, "retry_in", wait)
+			}
+		}
+		m.mu.Lock()
+		m.nextAttempt = m.now().Add(wait).UTC()
+		m.mu.Unlock()
+		if !timer.Stop() {
+			select {
+			case <-timer.C:
+			default:
+			}
+		}
+		timer.Reset(wait)
+	}
+}
+
+// Kick asks Run to renew now, even if the certificate isn't due.
+func (m *Manager) Kick() {
+	m.mu.Lock()
+	m.force, m.failures = true, 0
+	m.mu.Unlock()
+	select {
+	case m.kick <- struct{}{}:
+	default:
+	}
+}
+
+// ErrBusy is returned while another order runs.
+var ErrBusy = errors.New("a certificate order is already running")
+
+// Renew gets a new certificate now: the preflight check, then the order.
+func (m *Manager) Renew(ctx context.Context) error {
+	if !m.lego.TryLock() {
+		return ErrBusy
+	}
+	defer m.lego.Unlock()
+	m.mu.Lock()
+	m.working, m.lastAttempt = true, m.now().UTC()
+	m.mu.Unlock()
+	err := m.renew(ctx)
+	m.mu.Lock()
+	m.working, m.force = false, false
+	if err != nil {
+		m.lastErr = err.Error()
+	} else {
+		m.lastErr, m.failures = "", 0
+	}
+	m.mu.Unlock()
+	return err
+}
+
+func (m *Manager) renew(ctx context.Context) error {
+	c := m.Preflight(ctx)
+	if !c.OK {
+		for _, s := range c.Steps {
+			if s.Status == "fail" {
+				return fmt.Errorf("the check before asking the CA failed at %s: %s", strings.ToLower(s.Name), s.Detail)
+			}
+		}
+	}
+	m.log.Info("the check passed; asking the CA for a certificate", "domain", m.cfg.ACME.Domain, "directory", m.cfg.ACME.DirectoryURL())
+	res, err := m.obtain(ctx)
+	if err != nil {
+		return err
+	}
+	s := stored{Domain: m.cfg.ACME.Domain, Directory: m.cfg.ACME.DirectoryURL(), CertPEM: string(res.Certificate), KeyPEM: string(res.PrivateKey), Obtained: m.now().UTC()}
+	if err := m.use(s); err != nil {
+		return fmt.Errorf("the CA's certificate can't be used: %w", err)
+	}
+	b, _ := json.Marshal(s)
+	if err := m.store.SetMeta(ctx, metaCert, string(b)); err != nil {
+		return err
+	}
+	m.mu.Lock()
+	na := m.leaf.NotAfter
+	m.mu.Unlock()
+	m.log.Info("got a new certificate for the web UI", "domain", s.Domain, "until", na.Format(time.RFC3339), "staging", m.cfg.ACME.Staging())
+	return nil
+}
+
+// ---- ACME
+
+type account struct {
+	Directory string `json:"directory"`
+	Email     string `json:"email,omitempty"`
+	URI       string `json:"uri,omitempty"`
+	KeyPEM    string `json:"key_pem"`
+}
+
+type legoUser struct {
+	email string
+	reg   *registration.Resource
+	key   crypto.PrivateKey
+}
+
+func (u *legoUser) GetEmail() string                        { return u.email }
+func (u *legoUser) GetRegistration() *registration.Resource { return u.reg }
+func (u *legoUser) GetPrivateKey() crypto.PrivateKey        { return u.key }
+
+func (m *Manager) httpClient() (*http.Client, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
+	if f := m.cfg.ACME.CABundle; f != "" {
+		pemData, err := os.ReadFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("tls.acme.ca_bundle: %w", err)
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil || pool == nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(pemData) {
+			return nil, fmt.Errorf("tls.acme.ca_bundle: no certificates in %s", f)
+		}
+		tr.TLSClientConfig.RootCAs = pool
+	}
+	return &http.Client{Timeout: 2 * time.Minute, Transport: tr}, nil
+}
+
+// client returns a lego client for the account, registering it if needed.
+func (m *Manager) client(ctx context.Context) (*lego.Client, error) {
+	dir := m.cfg.ACME.DirectoryURL()
+	hc, err := m.httpClient()
+	if err != nil {
+		return nil, err
+	}
+	user := &legoUser{email: m.cfg.ACME.Email}
+	var a account
+	if raw, _ := m.store.GetMeta(ctx, metaAccount); raw != "" && json.Unmarshal([]byte(raw), &a) == nil && a.Directory == dir && a.KeyPEM != "" {
+		b, _ := pem.Decode([]byte(a.KeyPEM))
+		if b == nil {
+			return nil, errors.New("the stored ACME account key isn't PEM")
+		}
+		if user.key, err = x509.ParsePKCS8PrivateKey(b.Bytes); err != nil {
+			return nil, fmt.Errorf("the stored ACME account key: %w", err)
+		}
+		if a.URI != "" {
+			user.reg = &registration.Resource{URI: a.URI}
+		}
+	} else {
+		if user.key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader); err != nil {
+			return nil, err
+		}
+	}
+	cfg := lego.NewConfig(user)
+	cfg.CADirURL, cfg.HTTPClient, cfg.UserAgent = dir, hc, "housecarl/"+m.cfg.Version
+	cfg.Certificate.Timeout = 90 * time.Second
+	c, err := lego.NewClient(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("the ACME server %s: %w", dir, err)
+	}
+	if user.reg != nil {
+		return c, nil
+	}
+	if !m.cfg.ACME.AgreeTOS {
+		tos := c.GetToSURL()
+		if tos == "" {
+			tos = "on the CA's website"
+		}
+		return nil, fmt.Errorf("Housecarl registers an account with %s only after you accept its terms of service (%s): set tls.acme.agree_tos: true in housecarl.yaml and restart housecarld", dir, tos)
+	}
+	reg, err := c.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: true})
+	if err != nil {
+		return nil, fmt.Errorf("registering the ACME account: %w", err)
+	}
+	user.reg = reg
+	der, err := x509.MarshalPKCS8PrivateKey(user.key)
+	if err != nil {
+		return nil, err
+	}
+	b, _ := json.Marshal(account{Directory: dir, Email: user.email, URI: reg.URI,
+		KeyPEM: string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))})
+	if err := m.store.SetMeta(ctx, metaAccount, string(b)); err != nil {
+		return nil, err
+	}
+	m.log.Info("ACME account registered", "directory", dir, "account", reg.URI)
+	return c, nil
+}
+
+// provider publishes the challenge through Jarl.
+type provider struct {
+	m   *Manager
+	ctx context.Context
+}
+
+func (p *provider) set(domain, keyAuth string, set bool) error {
+	info := dns01.GetChallengeInfo(domain, keyAuth)
+	ctx := p.ctx
+	if !set {
+		ctx = context.WithoutCancel(ctx)
+	}
+	all, err := p.m.jarl.Zones(ctx)
+	if err != nil {
+		return err
+	}
+	zones := jarl.ZonesFor(info.EffectiveFQDN, all, "")
+	if len(zones) == 0 {
+		return fmt.Errorf("no primary zone in Jarl holds %s", strings.TrimSuffix(info.EffectiveFQDN, "."))
+	}
+	return p.m.jarl.SetTXT(ctx, info.EffectiveFQDN, info.Value, zones, set, false, "Housecarl's certificate")
+}
+
+func (p *provider) Present(domain, token, keyAuth string) error { return p.set(domain, keyAuth, true) }
+func (p *provider) CleanUp(domain, token, keyAuth string) error { return p.set(domain, keyAuth, false) }
+
+// Timeout is how long lego waits for the record to reach the name servers.
+func (p *provider) Timeout() (timeout, interval time.Duration) {
+	return 5 * time.Minute, 5 * time.Second
+}
+
+func (m *Manager) obtain(ctx context.Context) (*certificate.Resource, error) {
+	c, err := m.client(ctx)
+	if err != nil {
+		return nil, err
+	}
+	opts := []dns01.ChallengeOption{dns01.AddDNSTimeout(10 * time.Second)}
+	if len(m.cfg.ACME.Resolvers) > 0 {
+		opts = append(opts, dns01.AddRecursiveNameservers(m.cfg.ACME.Resolvers))
+	}
+	if !m.cfg.ACME.CheckPropagation {
+		opts = append(opts, dns01.DisableAuthoritativeNssPropagationRequirement())
+	}
+	if err := c.Challenge.SetDNS01Provider(&provider{m: m, ctx: ctx}, opts...); err != nil {
+		return nil, err
+	}
+	res, err := c.Certificate.Obtain(certificate.ObtainRequest{Domains: []string{m.cfg.ACME.Domain}, Bundle: true})
+	if err != nil {
+		return nil, cleanLegoError(err)
+	}
+	return res, nil
+}
+
+// cleanLegoError shortens lego's errors (as Jarl does): "[name] acme:
+// error: 403 :: urn:ietf:params:acme:error:unauthorized :: MSG" becomes
+// "name: the CA refused (unauthorized): MSG".
+func cleanLegoError(err error) error {
+	msg := strings.TrimSpace(strings.Replace(err.Error(), "error: one or more domains had a problem:", "", 1))
+	var out []string
+	for _, line := range strings.Split(msg, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		domain := ""
+		for strings.HasPrefix(line, "[") {
+			end := strings.Index(line, "] ")
+			if end < 0 {
+				break
+			}
+			if domain == "" {
+				domain = line[1:end]
+			}
+			line = line[end+2:]
+		}
+		for _, p := range []string{"acme: error presenting token: ", "acme: error: ", "acme: "} {
+			line = strings.TrimPrefix(line, p)
+		}
+		if mm := problemRe.FindStringSubmatch(line); mm != nil {
+			line = "the CA refused (" + mm[2] + "): " + mm[3]
+		}
+		if domain != "" {
+			line = domain + ": " + line
+		}
+		out = append(out, line)
+	}
+	if len(out) == 0 {
+		return err
+	}
+	return errors.New(strings.Join(out, "; "))
+}
+
+var problemRe = regexp.MustCompile(`^(\d+) :: urn:ietf:params:acme:error:(\w+) :: (.*)$`)

internal/client/client.go

@@ -0,0 +1,205 @@
+// Package client is a small Go client for Housecarl's REST API, used by the
+// housecarl CLI.
+package client
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"housecarl/internal/api"
+	"housecarl/internal/version"
+)
+
+// Client talks to a Housecarl server.
+type Client struct {
+	BaseURL string
+	Token   string
+	HTTP    *http.Client
+	// DryRun previews changes instead of making them.
+	DryRun bool
+	// Changeset stages configuration changes in that changeset instead
+	// of applying them (X-Housecarl-Changeset).
+	Changeset int64
+	// BeforeStage, if set, runs once before the first change is staged,
+	// and can stop it (the CLI checks HOUSECARL_CHANGESET's changeset).
+	BeforeStage func() error
+	staged      bool
+}
+
+// Staged reports whether a request is a configuration change that a
+// changeset takes: objects, settings and undo, and the changes that can't
+// be staged (import, takeover), which the server then refuses rather than
+// apply them. Changesets' own calls and dynamic set entries aren't.
+func Staged(method, path string) bool {
+	if method == http.MethodGet || strings.Contains(path, "/elements") {
+		return false
+	}
+	for _, p := range []string{"/zones", "/sets", "/services", "/rules", "/forwards", "/snats", "/redirects", "/settings", "/history/", "/import/", "/takeover"} {
+		if strings.HasPrefix(path, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// Options configure New.
+type Options struct {
+	Server   string // https://... or unix:///run/housecarl/housecarl.sock
+	Token    string
+	Insecure bool   // skip TLS certificate checks
+	CACert   string // PEM file with an extra CA to trust
+	Timeout  time.Duration
+}
+
+// New makes a client.
+func New(o Options) (*Client, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	base := strings.TrimSuffix(o.Server, "/")
+	if path, ok := SocketPath(o.Server); ok {
+		tr.Proxy = nil
+		tr.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, "unix", path)
+		}
+		base = "http://housecarld"
+	} else if u, err := url.Parse(o.Server); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+		return nil, fmt.Errorf("invalid server URL %q (use e.g. https://firewall.example.com:8070, or unix:///run/housecarl/housecarl.sock)", o.Server)
+	}
+	tc := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Insecure} //nolint:gosec // the user's choice
+	if o.CACert != "" {
+		pem, err := os.ReadFile(o.CACert)
+		if err != nil {
+			return nil, err
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil || pool == nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("no certificates in %s", o.CACert)
+		}
+		tc.RootCAs = pool
+	}
+	tr.TLSClientConfig = tc
+	timeout := o.Timeout
+	if timeout == 0 {
+		timeout = 2 * time.Minute
+	}
+	return &Client{BaseURL: base, Token: o.Token, HTTP: &http.Client{Transport: tr, Timeout: timeout}}, nil
+}
+
+// SocketPath returns the path of a unix:// URL.
+func SocketPath(server string) (string, bool) {
+	path, ok := strings.CutPrefix(server, "unix://")
+	if !ok || !strings.HasPrefix(path, "/") {
+		return "", false
+	}
+	return path, true
+}
+
+// Error is an API error.
+type Error struct {
+	Status int
+	Info   api.ErrorInfo
+}
+
+func (e *Error) Error() string {
+	if e.Info.Message != "" {
+		return e.Info.Message
+	}
+	return fmt.Sprintf("HTTP %d", e.Status)
+}
+
+// Do sends a request: body as JSON, the JSON answer into out (if not nil;
+// a *string gets the answer as text).
+// Changes get dry_run=1 when the client's DryRun is set.
+func (c *Client) Do(ctx context.Context, method, path string, query url.Values, body, out any) error {
+	if c.DryRun && method != http.MethodGet {
+		if query == nil {
+			query = url.Values{}
+		}
+		query.Set("dry_run", "1")
+	}
+	stage := c.Changeset != 0 && Staged(method, path)
+	if stage && !c.staged && c.BeforeStage != nil {
+		if err := c.BeforeStage(); err != nil {
+			return err
+		}
+	}
+	u := c.BaseURL + api.Prefix + path
+	if len(query) > 0 {
+		u += "?" + query.Encode()
+	}
+	var rd io.Reader
+	if body != nil {
+		b, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		rd = bytes.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, u, rd)
+	if err != nil {
+		return err
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "housecarl-cli/"+version.Short())
+	if c.Token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.Token)
+	}
+	if stage {
+		req.Header.Set("X-Housecarl-Changeset", strconv.FormatInt(c.Changeset, 10))
+		c.staged = true
+	}
+	resp, err := c.HTTP.Do(req)
+	if err != nil {
+		if c.BaseURL == "http://housecarld" {
+			return fmt.Errorf("can't reach housecarld on its local socket: %w (is housecarld running? 'systemctl status housecarl')", err)
+		}
+		return err
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		e := &Error{Status: resp.StatusCode}
+		var eb api.ErrorBody
+		if json.Unmarshal(b, &eb) == nil && eb.Error.Message != "" {
+			e.Info = eb.Error
+		} else {
+			e.Info.Message = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
+		}
+		return e
+	}
+	if text, ok := out.(*string); ok {
+		*text = string(b)
+		return nil
+	}
+	if out != nil && len(b) > 0 {
+		if err := json.Unmarshal(b, out); err != nil {
+			return fmt.Errorf("reading the answer: %w", err)
+		}
+	}
+	return nil
+}
+
+// Get is Do for GET requests.
+func (c *Client) Get(ctx context.Context, path string, query url.Values, out any) error {
+	return c.Do(ctx, http.MethodGet, path, query, nil, out)
+}

internal/config/config.go

@@ -0,0 +1,493 @@
+// Package config loads housecarld's configuration file. It only says how
+// the daemon runs (where it listens, its files, logging); the firewall's
+// configuration itself is in the database.
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/netip"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"go.yaml.in/yaml/v3"
+)
+
+// DefaultPath is where housecarld looks for its configuration.
+const DefaultPath = "/etc/housecarl/housecarl.yaml"
+
+// Config is the daemon configuration.
+type Config struct {
+	// Listen are the web UI's and API's addresses.
+	Listen         []string `yaml:"listen"`
+	TLS            TLS      `yaml:"tls"`
+	TrustedProxies []string `yaml:"trusted_proxies"`
+	// Socket is a local unix socket for the API, where root and
+	// housecarld's own user need no token ("" for none).
+	Socket   string  `yaml:"socket"`
+	Database string  `yaml:"database"`
+	Log      Log     `yaml:"log"`
+	Session  Session `yaml:"session"`
+	Backup   Backup  `yaml:"backup"`
+	// Jarl is the Jarl server Housecarl uses for its web UI's
+	// certificate (docs/certificates.md).
+	Jarl Jarl `yaml:"jarl"`
+	// Ruleset is where the confirmed ruleset is saved for the boot unit
+	// (housecarl-ruleset.service loads it before the network comes up).
+	Ruleset string `yaml:"ruleset"`
+	// Stats keeps traffic, hits and drops over time (docs/stats.md).
+	Stats Stats `yaml:"stats"`
+	// Shadow loads the ruleset without letting it block, reject or
+	// translate anything, beside the firewall in force (docs/shadow.md).
+	Shadow Shadow `yaml:"shadow"`
+	// Names shows inside addresses' names, by reverse lookups
+	// (docs/configuration.md).
+	Names Names `yaml:"names"`
+}
+
+// Names configures the reverse lookups that put names beside inside
+// addresses in the web UI and hc.
+type Names struct {
+	Enabled bool `yaml:"enabled"`
+	// Resolver is a DNS server ("10.10.0.1:53"; "": the system's).
+	Resolver string `yaml:"resolver"`
+	// Cache is how long a name is kept.
+	Cache Duration `yaml:"cache"`
+}
+
+// Shadow is shadow mode: Housecarl's ruleset is loaded and counts, logs
+// and traces what it would do, while another firewall decides.
+type Shadow struct {
+	Enabled bool `yaml:"enabled"`
+	// Position is where Housecarl's filter chains run: after the other
+	// firewall's (it sees what that one lets through, so what it would
+	// drop is what switching would break) or before (it sees everything).
+	Position string `yaml:"position"`
+}
+
+// Shadow positions.
+const (
+	ShadowAfter  = "after"
+	ShadowBefore = "before"
+)
+
+// Stats configures statistics: a database of their own, aggregates
+// only, pruned by age.
+type Stats struct {
+	Enabled   bool           `yaml:"enabled"`
+	Database  string         `yaml:"database"`
+	Retention StatsRetention `yaml:"retention"`
+	// TopN is how many entries the top lists keep per hour and day.
+	TopN int `yaml:"top_n"`
+}
+
+// StatsRetention is how long each resolution is kept.
+type StatsRetention struct {
+	Minute Duration `yaml:"minute"`
+	Hour   Duration `yaml:"hour"`
+	Day    Duration `yaml:"day"`
+}
+
+// TLS is the web UI's certificate: from files, or obtained through ACME
+// with Jarl publishing the DNS-01 challenge (docs/certificates.md).
+type TLS struct {
+	CertFile string `yaml:"cert_file"`
+	KeyFile  string `yaml:"key_file"`
+	ACME     ACME   `yaml:"acme"`
+}
+
+// ACME obtains the web UI's certificate (docs/certificates.md).
+type ACME struct {
+	// Domain is the certificate's name ("" turns ACME off).
+	Domain string `yaml:"domain"`
+	Email  string `yaml:"email"`
+	// Directory is letsencrypt, letsencrypt-staging (the default, for a
+	// first certificate browsers don't trust) or an ACME directory URL.
+	Directory string `yaml:"directory"`
+	// AgreeTOS says you accept the CA's terms of service; no account is
+	// registered without it.
+	AgreeTOS bool `yaml:"agree_tos"`
+	// Resolvers are public recursive resolvers, to see the challenge as
+	// the CA would.
+	Resolvers []string `yaml:"resolvers"`
+	// CheckPropagation also asks each of the zone's public name servers
+	// directly before the CA is told to look.
+	CheckPropagation bool `yaml:"check_propagation"`
+	// CABundle is a PEM file of extra CAs to trust for the ACME server
+	// (a private CA such as step-ca, or Pebble in tests).
+	CABundle string `yaml:"ca_bundle"`
+}
+
+// Directory URLs for the names tls.acme.directory accepts.
+const (
+	LetsEncrypt        = "https://acme-v02.api.letsencrypt.org/directory"
+	LetsEncryptStaging = "https://acme-staging-v02.api.letsencrypt.org/directory"
+)
+
+// DirectoryURL is the ACME directory's URL.
+func (a ACME) DirectoryURL() string {
+	switch a.Directory {
+	case "", "letsencrypt-staging":
+		return LetsEncryptStaging
+	case "letsencrypt":
+		return LetsEncrypt
+	}
+	return a.Directory
+}
+
+// Staging reports whether the directory is Let's Encrypt's staging one.
+func (a ACME) Staging() bool { return a.DirectoryURL() == LetsEncryptStaging }
+
+// Jarl is a Jarl server and an API token for it.
+type Jarl struct {
+	// Server is Jarl's URL ("" when Housecarl doesn't use Jarl).
+	Server string `yaml:"server"`
+	// Token, or TokenFile holding it (better: the file can be readable by
+	// the housecarl user alone).
+	Token     string `yaml:"token"`
+	TokenFile string `yaml:"token_file"`
+	// CACert is a PEM file with a CA to trust for Jarl's certificate.
+	CACert   string `yaml:"ca_cert"`
+	Insecure bool   `yaml:"insecure"`
+	// Refresh is how often the Jarl sets' names are read (docs/jarl.md).
+	Refresh Duration `yaml:"refresh"`
+}
+
+// Enabled reports whether a Jarl server is configured.
+func (j Jarl) Enabled() bool { return j.Server != "" }
+
+// ReadToken returns the token, from the setting or the file.
+func (j Jarl) ReadToken() (string, error) {
+	if j.TokenFile == "" {
+		return j.Token, nil
+	}
+	b, err := os.ReadFile(j.TokenFile)
+	if err != nil {
+		return "", fmt.Errorf("jarl.token_file: %w", err)
+	}
+	return strings.TrimSpace(string(b)), nil
+}
+
+// Log configures logging to stderr (the journal under systemd).
+type Log struct {
+	Level  string `yaml:"level"`  // debug, info, warn, error
+	Format string `yaml:"format"` // text or json
+}
+
+// Session configures web UI logins.
+type Session struct {
+	Lifetime    Duration `yaml:"lifetime"`
+	IdleTimeout Duration `yaml:"idle_timeout"`
+	// CookieSecure is auto (secure over HTTPS), always or never.
+	CookieSecure string `yaml:"cookie_secure"`
+}
+
+// Backup configures 'housecarld backup'.
+type Backup struct {
+	Dir       string   `yaml:"dir"`
+	Retention Duration `yaml:"retention"`
+	// KeepAtLeast backups are kept however old they are.
+	KeepAtLeast int `yaml:"keep_at_least"`
+}
+
+// Duration is a time.Duration written as "15s" or "12h"; days ("30d")
+// work too.
+type Duration time.Duration
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (d *Duration) UnmarshalYAML(n *yaml.Node) error {
+	var s string
+	if err := n.Decode(&s); err != nil {
+		return err
+	}
+	v, err := ParseDuration(s)
+	if err != nil {
+		return err
+	}
+	*d = Duration(v)
+	return nil
+}
+
+// ParseDuration parses a Go duration, also accepting days ("30d").
+func ParseDuration(s string) (time.Duration, error) {
+	s = strings.TrimSpace(s)
+	if n, ok := strings.CutSuffix(s, "d"); ok {
+		var days int
+		if _, err := fmt.Sscanf(n, "%d", &days); err != nil || fmt.Sprint(days) != n {
+			return 0, fmt.Errorf("invalid duration %q", s)
+		}
+		return time.Duration(days) * 24 * time.Hour, nil
+	}
+	return time.ParseDuration(s)
+}
+
+// Default returns the defaults for Debian/Ubuntu.
+func Default() *Config {
+	return &Config{
+		Listen:         []string{"127.0.0.1:8070"},
+		TrustedProxies: []string{},
+		Ruleset:        "/var/lib/housecarl/ruleset.nft",
+		Socket:         "/run/housecarl/housecarl.sock",
+		Database:       "/var/lib/housecarl/housecarl.db",
+		Log:            Log{Level: "info", Format: "text"},
+		Session:        Session{Lifetime: Duration(12 * time.Hour), IdleTimeout: Duration(2 * time.Hour), CookieSecure: "auto"},
+		Backup:         Backup{Dir: "/var/backups/housecarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7},
+		Stats: Stats{Enabled: true, Database: "/var/lib/housecarl/stats.db", TopN: 100,
+			Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)}},
+		Shadow: Shadow{Position: ShadowAfter},
+		Names:  Names{Enabled: true, Cache: Duration(10 * time.Minute)},
+		Jarl:   Jarl{Refresh: Duration(time.Minute)},
+		TLS:    TLS{ACME: ACME{Directory: "letsencrypt-staging", Resolvers: []string{"1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"}, CheckPropagation: true}},
+	}
+}
+
+// Load reads the file at path over the defaults. A missing file is fine
+// when allowMissing is set.
+func Load(path string, allowMissing bool) (*Config, error) {
+	c := Default()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) && allowMissing {
+			return c, c.check()
+		}
+		return nil, err
+	}
+	dec := yaml.NewDecoder(bytes.NewReader(b))
+	dec.KnownFields(true)
+	if err := dec.Decode(c); err != nil && err.Error() != "EOF" {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	if err := c.check(); err != nil {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	return c, nil
+}
+
+func (c *Config) check() error {
+	if len(c.Listen) == 0 {
+		return fmt.Errorf("listen: at least one address is needed")
+	}
+	if c.Socket != "" && !filepath.IsAbs(c.Socket) {
+		return fmt.Errorf("socket must be an absolute path (or \"\" for none)")
+	}
+	if len(c.Socket) > 107 {
+		return fmt.Errorf("socket: the path is %d bytes long; unix sockets allow at most 107", len(c.Socket))
+	}
+	if (c.TLS.CertFile == "") != (c.TLS.KeyFile == "") {
+		return fmt.Errorf("tls: set both cert_file and key_file")
+	}
+	if err := c.checkACME(); err != nil {
+		return err
+	}
+	for _, p := range c.TrustedProxies {
+		if _, err := ParsePrefix(p); err != nil {
+			return fmt.Errorf("trusted_proxies: %v", err)
+		}
+	}
+	if c.Ruleset != "" && !filepath.IsAbs(c.Ruleset) {
+		return fmt.Errorf("ruleset must be an absolute path (or \"\" for none)")
+	}
+	if c.Database == "" || !filepath.IsAbs(c.Database) {
+		return fmt.Errorf("database must be an absolute path")
+	}
+	if c.Backup.Dir != "" && !filepath.IsAbs(c.Backup.Dir) {
+		return fmt.Errorf("backup.dir must be an absolute path")
+	}
+	switch c.Session.CookieSecure {
+	case "auto", "always", "never":
+	default:
+		return fmt.Errorf("session.cookie_secure must be auto, always or never")
+	}
+	switch c.Log.Format {
+	case "text", "json":
+	default:
+		return fmt.Errorf("log.format must be text or json")
+	}
+	switch c.Log.Level {
+	case "debug", "info", "warn", "error":
+	default:
+		return fmt.Errorf("log.level must be debug, info, warn or error")
+	}
+	if c.Shadow.Position != ShadowAfter && c.Shadow.Position != ShadowBefore {
+		return fmt.Errorf("shadow.position must be after or before")
+	}
+	if c.Names.Resolver != "" {
+		if _, err := netip.ParseAddrPort(c.Names.Resolver); err != nil {
+			return fmt.Errorf("names.resolver must be an address and port, such as 10.10.0.1:53")
+		}
+	}
+	if c.Names.Cache <= 0 {
+		return fmt.Errorf("names.cache must be a duration, such as 10m")
+	}
+	if c.Jarl.Refresh < Duration(10*time.Second) {
+		return fmt.Errorf("jarl.refresh must be a duration of 10s or more, such as 1m")
+	}
+	return nil
+}
+
+func (c *Config) checkACME() error {
+	j := c.Jarl
+	if j.Server != "" {
+		if u, err := url.Parse(j.Server); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+			return fmt.Errorf("jarl.server %q: give Jarl's URL, e.g. https://dns.example.com:8053", j.Server)
+		}
+		if j.Token == "" && j.TokenFile == "" {
+			return fmt.Errorf("jarl: set token or token_file (a Jarl API token, docs/jarl.md)")
+		}
+		if j.Token != "" && j.TokenFile != "" {
+			return fmt.Errorf("jarl: set token or token_file, not both")
+		}
+	}
+	a := c.TLS.ACME
+	if a.Domain == "" {
+		return nil
+	}
+	if c.TLS.CertFile != "" {
+		return fmt.Errorf("tls: cert_file and acme can't both be set; choose one")
+	}
+	if !j.Enabled() {
+		return fmt.Errorf("tls.acme needs Jarl to publish the challenge: set jarl.server and its token (docs/certificates.md)")
+	}
+	d := strings.TrimSuffix(strings.ToLower(a.Domain), ".")
+	if strings.HasPrefix(d, "*.") || strings.Count(d, ".") < 1 || strings.ContainsAny(d, " /:") {
+		return fmt.Errorf("tls.acme.domain %q: give the full name the web UI is reached by, e.g. firewall.example.com", a.Domain)
+	}
+	if u := a.DirectoryURL(); !strings.HasPrefix(u, "https://") {
+		return fmt.Errorf("tls.acme.directory %q: use letsencrypt, letsencrypt-staging or an https:// ACME directory URL", a.Directory)
+	}
+	for _, r := range a.Resolvers {
+		if _, _, err := net.SplitHostPort(r); err != nil {
+			return fmt.Errorf("tls.acme.resolvers: %q must be HOST:PORT, e.g. 1.1.1.1:53", r)
+		}
+	}
+	return nil
+}
+
+// ParsePrefix parses an address or a prefix.
+func ParsePrefix(s string) (netip.Prefix, error) {
+	if strings.Contains(s, "/") {
+		return netip.ParsePrefix(s)
+	}
+	a, err := netip.ParseAddr(s)
+	if err != nil {
+		return netip.Prefix{}, err
+	}
+	return netip.PrefixFrom(a, a.BitLen()), nil
+}
+
+// Proxies returns the trusted proxies.
+func (c *Config) Proxies() []netip.Prefix {
+	var out []netip.Prefix
+	for _, p := range c.TrustedProxies {
+		if pf, err := ParsePrefix(p); err == nil {
+			out = append(out, pf.Masked())
+		}
+	}
+	return out
+}
+
+// Example is a commented example configuration with the defaults.
+const Example = `# Housecarl daemon configuration (/etc/housecarl/housecarl.yaml).
+# Every setting is optional; the values shown are the defaults. The
+# firewall's configuration (zones, sets, rules, NAT) is not here: it is in
+# the database, changed with hc, the API or the web UI.
+
+# Addresses the web UI and API listen on. Keep this off the internet:
+# bind to localhost or a management network, and use TLS (below) or a
+# TLS-terminating reverse proxy.
+listen:
+  - "127.0.0.1:8070"
+
+# HTTPS for the web UI and API: a certificate from files you renew yourself
+# (reread when they change)...
+# tls:
+#   cert_file: /etc/housecarl/tls/fullchain.pem
+#   key_file: /etc/housecarl/tls/privkey.pem
+#
+# ...or one Housecarl obtains and renews itself from Let's Encrypt, with Jarl
+# publishing the DNS-01 challenge (needs jarl: below; docs/certificates.md).
+# Start with the staging directory: its certificate isn't trusted by
+# browsers, but it proves everything works without Let's Encrypt's rate
+# limits. Then set directory: letsencrypt.
+tls:
+  acme:
+    domain: ""                       # e.g. firewall.example.com; "" is off
+    email: ""
+    directory: letsencrypt-staging   # letsencrypt, or an ACME directory URL
+    agree_tos: false                 # you accept the CA's terms of service
+    # Public resolvers, to see the challenge as the CA does.
+    resolvers: ["1.1.1.1:53", "9.9.9.9:53", "8.8.8.8:53"]
+    # Also ask the zone's public name servers directly.
+    check_propagation: true
+    ca_bundle: ""                    # extra CA for a private ACME server
+
+# Jarl, the BIND manager, publishes the certificate's DNS-01 challenge
+# (docs/certificates.md). Give Housecarl a token that may only write its
+# challenge record; keep it in a file only the housecarl user can read.
+# jarl:
+#   server: https://dns.example.com:8053
+#   token_file: /etc/housecarl/jarl.token
+#   ca_cert: ""                      # a CA to trust for Jarl's certificate
+#   insecure: false
+
+# The confirmed ruleset, saved for housecarl-ruleset.service, which loads
+# it at boot before the network comes up (and before housecarld starts).
+ruleset: /var/lib/housecarl/ruleset.nft
+
+# Reverse proxies whose X-Forwarded-For and X-Forwarded-Proto are
+# believed.
+trusted_proxies: []
+
+# A local socket for the API: root and housecarld's own user use it without
+# a token ('sudo hc status' just works), and changes made through it need
+# no confirmation. "" turns it off.
+socket: /run/housecarl/housecarl.sock
+
+database: /var/lib/housecarl/housecarl.db
+
+log:
+  level: info     # debug, info, warn or error
+  format: text    # text or json
+
+session:
+  lifetime: 12h
+  idle_timeout: 2h
+  cookie_secure: auto   # auto (secure over HTTPS), always or never
+
+# 'housecarld backup' (the daily housecarl-backup.timer) writes copies of the
+# database here and deletes those older than retention, keeping at least
+# keep_at_least.
+backup:
+  dir: /var/backups/housecarl
+  retention: 30d
+  keep_at_least: 7
+
+# Statistics (docs/stats.md): traffic per interface and zone, hits and
+# drops, blocked sources, connections, quotas, kept over time in a database
+# of their own (aggregates only). Deleting it is always safe. Traffic per
+# device is a firewall setting: hc settings set --stats-per-device on.
+stats:
+  enabled: true
+  database: /var/lib/housecarl/stats.db
+  retention:
+    minute: 7d     # per-minute figures
+    hour: 90d      # hourly figures and top lists
+    day: 730d      # daily figures and top lists
+  top_n: 100       # entries kept per top list, per hour and day
+
+# Shadow mode (docs/shadow.md): Housecarl's ruleset is loaded beside the
+# firewall you have now, but blocks, rejects and translates nothing. Its
+# counters, log and tracer show what it would do. No change is on
+# probation, and takeovers and closing connections are refused. housecarld
+# reads this when it starts: restart it to switch.
+shadow:
+  enabled: false
+  # after: Housecarl's filter chains run after the other firewall's, and
+  # see only what it lets through: what Housecarl would drop is what
+  # switching over would break. before: they see every packet.
+  position: after
+`

internal/config/config_test.go

@@ -0,0 +1,50 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestExampleIsTheDefaults(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "housecarl.yaml")
+	os.WriteFile(path, []byte(Example), 0o644)
+	c, err := Load(path, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(c, Default()) {
+		t.Errorf("example differs from the defaults:\n%+v\n%+v", c, Default())
+	}
+}
+
+func TestErrors(t *testing.T) {
+	cases := map[string]string{
+		"listen: []":                                "listen",
+		"database: relative.db":                     "database",
+		"session: {cookie_secure: x}":               "cookie_secure",
+		"tls: {cert_file: /a}":                      "tls",
+		"no_such_setting: 1":                        "no_such_setting",
+		"backup: {retention: 3x}":                   "duration",
+		"tls: {acme: {domain: dhcp.example.com}}":   "needs Jarl",
+		"jarl: {server: https://dns.example.com}":   "token or token_file",
+		"jarl: {server: dns.example.com, token: x}": "Jarl's URL",
+		"{tls: {cert_file: /a, key_file: /b, acme: {domain: d.example.com}}, jarl: {server: https://j, token: x}}": "can't both",
+		"{tls: {acme: {domain: '*.example.com'}}, jarl: {server: https://j, token: x}}":                            "full name",
+		"{tls: {acme: {domain: d.example.com, directory: http://x}}, jarl: {server: https://j, token: x}}":         "https://",
+		"ruleset: relative.nft": "ruleset",
+		"shadow: {position: x}": "shadow.position",
+	}
+	for yml, want := range cases {
+		path := filepath.Join(t.TempDir(), "s.yaml")
+		os.WriteFile(path, []byte(yml), 0o644)
+		if _, err := Load(path, false); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("%q: %v", yml, err)
+		}
+	}
+	if _, err := Load("/nonexistent/housecarl.yaml", true); err != nil {
+		t.Errorf("missing file allowed: %v", err)
+	}
+}

internal/conntrack/conntrack.go

@@ -0,0 +1,351 @@
+// Package conntrack lists and closes the kernel's tracked connections,
+// through ctnetlink (the netfilter netlink subsystem for connection
+// tracking): what the "established" built-in lets through, and what a
+// change of rules doesn't stop by itself.
+//
+// Closing a connection deletes its tracking entry. Its next packet is
+// then checked by the rules again, as a new connection would be: one the
+// rules now block is stopped, one they allow carries on (TCP connections
+// are picked up again mid-stream).
+package conntrack
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net/netip"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+
+	"housecarl/internal/api"
+)
+
+const (
+	subsysCtnetlink = 1
+	msgNew          = 0
+	msgGet          = 1
+	msgDelete       = 2
+
+	ctaTupleOrig     = 1
+	ctaTupleReply    = 2
+	ctaStatus        = 3
+	ctaProtoinfo     = 4
+	ctaTimeout       = 7
+	ctaMark          = 8
+	ctaCountersOrig  = 9
+	ctaCountersReply = 10
+	ctaID            = 12
+
+	ctaTupleIP    = 1
+	ctaTupleProto = 2
+
+	ctaIPv4Src = 1
+	ctaIPv4Dst = 2
+	ctaIPv6Src = 3
+	ctaIPv6Dst = 4
+
+	ctaProtoNum      = 1
+	ctaProtoSrcPort  = 2
+	ctaProtoDstPort  = 3
+	ctaProtoICMPID   = 4
+	ctaProtoICMPType = 5
+	ctaProtoICMPCode = 6
+	ctaProtoICMP6ID  = 7
+	ctaProtoICMP6Typ = 8
+	ctaProtoICMP6Cod = 9
+
+	ctaProtoinfoTCP      = 1
+	ctaProtoinfoTCPState = 1
+
+	ctaCountersPackets = 1
+	ctaCountersBytes   = 2
+
+	statusAssured = 1 << 2
+	statusSrcNAT  = 1 << 4
+	statusDstNAT  = 1 << 5
+
+	nlaNested   = 0x8000
+	nlaTypeMask = 0x3fff
+)
+
+var tcpStates = []string{"none", "syn-sent", "syn-recv", "established", "fin-wait", "close-wait", "last-ack", "time-wait", "close", "syn-sent2"}
+
+var protoNames = map[uint8]string{1: "icmp", 6: "tcp", 17: "udp", 58: "icmpv6", 132: "sctp", 136: "udplite", 33: "dccp", 47: "gre"}
+
+// Client talks to ctnetlink. The zero value works.
+type Client struct{}
+
+var seq atomic.Uint32
+
+// List returns every tracked connection, IPv4 and IPv6.
+func (Client) List(ctx context.Context) ([]api.Connection, error) {
+	msgs, err := request(ctx, msgGet, unix.NLM_F_REQUEST|unix.NLM_F_DUMP, unix.AF_UNSPEC, nil)
+	if err != nil {
+		return nil, err
+	}
+	out := []api.Connection{}
+	for _, m := range msgs {
+		if c, ok := parse(m); ok {
+			out = append(out, c)
+		}
+	}
+	return out, nil
+}
+
+// Close deletes a connection's tracking entry, found by its original
+// direction (addresses, protocol, ports or ICMP id). Gone already is fine.
+func (Client) Close(ctx context.Context, c api.Connection) error {
+	src, err1 := netip.ParseAddr(c.Source)
+	dst, err2 := netip.ParseAddr(c.Destination)
+	if err1 != nil || err2 != nil || src.Is4() != dst.Is4() {
+		return fmt.Errorf("connection %s -> %s: not a pair of addresses", c.Source, c.Destination)
+	}
+	num := protoNumber(c.Protocol)
+	if num == 0 {
+		return fmt.Errorf("connection %s -> %s: unknown protocol %q", c.Source, c.Destination, c.Protocol)
+	}
+	family, ipSrc, ipDst := uint8(unix.AF_INET), uint16(ctaIPv4Src), uint16(ctaIPv4Dst)
+	if src.Is6() {
+		family, ipSrc, ipDst = unix.AF_INET6, ctaIPv6Src, ctaIPv6Dst
+	}
+	ip := attr(ipSrc, src.AsSlice())
+	ip = append(ip, attr(ipDst, dst.AsSlice())...)
+	proto := attr(ctaProtoNum, []byte{num})
+	switch num {
+	case 1:
+		proto = append(proto, attr(ctaProtoICMPID, be16(uint16(c.ICMPID)))...)
+		proto = append(proto, attr(ctaProtoICMPType, []byte{uint8(c.ICMPType)})...)
+		proto = append(proto, attr(ctaProtoICMPCode, []byte{uint8(c.ICMPCode)})...)
+	case 58:
+		proto = append(proto, attr(ctaProtoICMP6ID, be16(uint16(c.ICMPID)))...)
+		proto = append(proto, attr(ctaProtoICMP6Typ, []byte{uint8(c.ICMPType)})...)
+		proto = append(proto, attr(ctaProtoICMP6Cod, []byte{uint8(c.ICMPCode)})...)
+	default:
+		proto = append(proto, attr(ctaProtoSrcPort, be16(uint16(c.SourcePort)))...)
+		proto = append(proto, attr(ctaProtoDstPort, be16(uint16(c.DestinationPort)))...)
+	}
+	tuple := attr(ctaTupleIP|nlaNested, ip)
+	tuple = append(tuple, attr(ctaTupleProto|nlaNested, proto)...)
+	_, err := request(ctx, msgDelete, unix.NLM_F_REQUEST|unix.NLM_F_ACK, family, attr(ctaTupleOrig|nlaNested, tuple))
+	if errors.Is(err, unix.ENOENT) {
+		return nil
+	}
+	return err
+}
+
+func protoNumber(name string) uint8 {
+	for n, s := range protoNames {
+		if s == name {
+			return n
+		}
+	}
+	return 0
+}
+
+// ---- netlink
+
+func request(ctx context.Context, msg uint16, flags uint16, family uint8, payload []byte) ([][]byte, error) {
+	fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_NETFILTER)
+	if err != nil {
+		return nil, err
+	}
+	defer unix.Close(fd)
+	if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil {
+		return nil, err
+	}
+	deadline := time.Now().Add(10 * time.Second)
+	if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
+		deadline = d
+	}
+	tv := unix.NsecToTimeval(time.Until(deadline).Nanoseconds())
+	unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv)
+	unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<22)
+
+	s := seq.Add(1)
+	b := make([]byte, unix.SizeofNlMsghdr+4, unix.SizeofNlMsghdr+4+len(payload))
+	binary.NativeEndian.PutUint16(b[4:], subsysCtnetlink<<8|msg)
+	binary.NativeEndian.PutUint16(b[6:], flags)
+	binary.NativeEndian.PutUint32(b[8:], s)
+	b[16] = family // nfgenmsg: family, version 0, res_id 0
+	b = append(b, payload...)
+	binary.NativeEndian.PutUint32(b[0:], uint32(len(b)))
+	if err := unix.Sendto(fd, b, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil {
+		return nil, err
+	}
+	var out [][]byte
+	buf := make([]byte, 1<<16)
+	for {
+		n, _, err := unix.Recvfrom(fd, buf, 0)
+		if err != nil {
+			if errors.Is(err, unix.EAGAIN) {
+				return nil, errors.New("no answer from connection tracking (is nf_conntrack loaded?)")
+			}
+			return nil, err
+		}
+		for off := 0; off+unix.SizeofNlMsghdr <= n; {
+			l := int(binary.NativeEndian.Uint32(buf[off:]))
+			typ := binary.NativeEndian.Uint16(buf[off+4:])
+			if l < unix.SizeofNlMsghdr || off+l > n {
+				return nil, errors.New("connection tracking: a malformed answer")
+			}
+			body := buf[off+unix.SizeofNlMsghdr : off+l]
+			switch typ {
+			case unix.NLMSG_DONE:
+				return out, nil
+			case unix.NLMSG_ERROR:
+				if len(body) >= 4 {
+					if code := int32(binary.NativeEndian.Uint32(body)); code != 0 {
+						return nil, syscall.Errno(-code)
+					}
+				}
+				return out, nil // the acknowledgement
+			default:
+				out = append(out, append([]byte(nil), body...))
+			}
+			off += (l + 3) &^ 3
+		}
+		if flags&unix.NLM_F_DUMP == 0 && flags&unix.NLM_F_ACK == 0 {
+			return out, nil
+		}
+	}
+}
+
+func attr(typ uint16, data []byte) []byte {
+	l := 4 + len(data)
+	b := make([]byte, (l+3)&^3)
+	binary.NativeEndian.PutUint16(b[0:], uint16(l))
+	binary.NativeEndian.PutUint16(b[2:], typ)
+	copy(b[4:], data)
+	return b
+}
+
+func be16(v uint16) []byte {
+	b := make([]byte, 2)
+	binary.BigEndian.PutUint16(b, v)
+	return b
+}
+
+// attrs splits netlink attributes: type (without the nested flag) → data.
+func attrs(b []byte) map[uint16][]byte {
+	m := map[uint16][]byte{}
+	for len(b) >= 4 {
+		l := int(binary.NativeEndian.Uint16(b[0:]))
+		typ := binary.NativeEndian.Uint16(b[2:]) & nlaTypeMask
+		if l < 4 || l > len(b) {
+			break
+		}
+		m[typ] = b[4:l]
+		b = b[min((l+3)&^3, len(b)):]
+	}
+	return m
+}
+
+// ---- parsing
+
+type tuple struct {
+	src, dst         netip.Addr
+	proto            uint8
+	sport, dport     uint16
+	icmpID           uint16
+	icmpType, icmpCd uint8
+}
+
+func parseTuple(b []byte) (tuple, bool) {
+	var t tuple
+	a := attrs(b)
+	ip := attrs(a[ctaTupleIP])
+	if v, ok := ip[ctaIPv4Src]; ok {
+		t.src, _ = netip.AddrFromSlice(v)
+		t.dst, _ = netip.AddrFromSlice(ip[ctaIPv4Dst])
+	} else if v, ok := ip[ctaIPv6Src]; ok {
+		t.src, _ = netip.AddrFromSlice(v)
+		t.dst, _ = netip.AddrFromSlice(ip[ctaIPv6Dst])
+	}
+	p := attrs(a[ctaTupleProto])
+	if v := p[ctaProtoNum]; len(v) == 1 {
+		t.proto = v[0]
+	}
+	if v := p[ctaProtoSrcPort]; len(v) == 2 {
+		t.sport = binary.BigEndian.Uint16(v)
+	}
+	if v := p[ctaProtoDstPort]; len(v) == 2 {
+		t.dport = binary.BigEndian.Uint16(v)
+	}
+	for _, k := range [][3]uint16{{ctaProtoICMPID, ctaProtoICMPType, ctaProtoICMPCode}, {ctaProtoICMP6ID, ctaProtoICMP6Typ, ctaProtoICMP6Cod}} {
+		if v := p[k[0]]; len(v) == 2 {
+			t.icmpID = binary.BigEndian.Uint16(v)
+		}
+		if v := p[k[1]]; len(v) == 1 {
+			t.icmpType = v[0]
+		}
+		if v := p[k[2]]; len(v) == 1 {
+			t.icmpCd = v[0]
+		}
+	}
+	return t, t.src.IsValid() && t.dst.IsValid()
+}
+
+// parse reads one connection (an nfgenmsg and its attributes).
+func parse(m []byte) (api.Connection, bool) {
+	if len(m) < 4 {
+		return api.Connection{}, false
+	}
+	a := attrs(m[4:])
+	orig, ok1 := parseTuple(a[ctaTupleOrig])
+	reply, ok2 := parseTuple(a[ctaTupleReply])
+	if !ok1 || !ok2 {
+		return api.Connection{}, false
+	}
+	c := api.Connection{
+		Family: "ipv4", Protocol: protoNames[orig.proto],
+		Source: orig.src.String(), SourcePort: int(orig.sport), Destination: orig.dst.String(), DestinationPort: int(orig.dport),
+		ICMPID: int(orig.icmpID), ICMPType: int(orig.icmpType), ICMPCode: int(orig.icmpCd),
+	}
+	if orig.src.Is6() {
+		c.Family = "ipv6"
+	}
+	if c.Protocol == "" {
+		c.Protocol = fmt.Sprint(orig.proto)
+	}
+	// NAT shows in the reply direction: it comes from where the packets
+	// really went, to where they really came from.
+	if reply.src != orig.dst || reply.sport != orig.dport {
+		c.DNATTo = reply.src.String()
+		if reply.sport != orig.dport {
+			c.DNATPort = int(reply.sport)
+		}
+	}
+	if reply.dst != orig.src || reply.dport != orig.sport {
+		c.SNATTo = reply.dst.String()
+	}
+	if v := a[ctaStatus]; len(v) == 4 {
+		st := binary.BigEndian.Uint32(v)
+		c.Assured = st&statusAssured != 0
+	}
+	if v := a[ctaTimeout]; len(v) == 4 {
+		c.Timeout = int(binary.BigEndian.Uint32(v))
+	}
+	if v := a[ctaMark]; len(v) == 4 {
+		c.Mark = binary.BigEndian.Uint32(v)
+	}
+	if v := a[ctaID]; len(v) == 4 {
+		c.ID = binary.BigEndian.Uint32(v)
+	}
+	if v := attrs(attrs(a[ctaProtoinfo])[ctaProtoinfoTCP])[ctaProtoinfoTCPState]; len(v) == 1 && int(v[0]) < len(tcpStates) {
+		c.State = tcpStates[v[0]]
+	}
+	for _, k := range []uint16{ctaCountersOrig, ctaCountersReply} {
+		ct := attrs(a[k])
+		if v := ct[ctaCountersPackets]; len(v) == 8 {
+			c.Packets += binary.BigEndian.Uint64(v)
+		}
+		if v := ct[ctaCountersBytes]; len(v) == 8 {
+			c.Bytes += binary.BigEndian.Uint64(v)
+		}
+	}
+	return c, true
+}

internal/conntrack/conntrack_test.go

@@ -0,0 +1,51 @@
+package conntrack
+
+import (
+	"encoding/binary"
+	"net/netip"
+	"testing"
+)
+
+func nested(typ uint16, parts ...[]byte) []byte {
+	var b []byte
+	for _, p := range parts {
+		b = append(b, p...)
+	}
+	return attr(typ|nlaNested, b)
+}
+
+func tupleAttr(typ uint16, src, dst string, proto uint8, sport, dport uint16) []byte {
+	s, d := netip.MustParseAddr(src), netip.MustParseAddr(dst)
+	return nested(typ,
+		nested(ctaTupleIP, attr(ctaIPv4Src, s.AsSlice()), attr(ctaIPv4Dst, d.AsSlice())),
+		nested(ctaTupleProto, attr(ctaProtoNum, []byte{proto}), attr(ctaProtoSrcPort, be16(sport)), attr(ctaProtoDstPort, be16(dport))))
+}
+
+// A port-forwarded TCP connection from the internet, as ctnetlink
+// describes it: the reply comes from the inside host.
+func TestParse(t *testing.T) {
+	status, timeout := make([]byte, 4), make([]byte, 4)
+	binary.BigEndian.PutUint32(status, statusAssured|statusDstNAT)
+	binary.BigEndian.PutUint32(timeout, 431999)
+	msg := []byte{2, 0, 0, 0} // nfgenmsg: AF_INET
+	msg = append(msg, tupleAttr(ctaTupleOrig, "198.51.100.7", "203.0.113.10", 6, 51432, 8443)...)
+	msg = append(msg, tupleAttr(ctaTupleReply, "192.168.1.10", "198.51.100.7", 6, 443, 51432)...)
+	msg = append(msg, attr(ctaStatus, status)...)
+	msg = append(msg, attr(ctaTimeout, timeout)...)
+	msg = append(msg, nested(ctaProtoinfo, nested(ctaProtoinfoTCP, attr(ctaProtoinfoTCPState, []byte{3})))...)
+	c, ok := parse(msg)
+	if !ok {
+		t.Fatal("not parsed")
+	}
+	if c.Protocol != "tcp" || c.Source != "198.51.100.7" || c.SourcePort != 51432 || c.Destination != "203.0.113.10" || c.DestinationPort != 8443 ||
+		c.DNATTo != "192.168.1.10" || c.DNATPort != 443 || c.SNATTo != "" || c.State != "established" || !c.Assured || c.Timeout != 431999 {
+		t.Errorf("parsed %+v", c)
+	}
+	// Masqueraded on the way out: the reply comes back to the WAN address.
+	msg = []byte{2, 0, 0, 0}
+	msg = append(msg, tupleAttr(ctaTupleOrig, "192.168.1.20", "198.51.100.9", 17, 5353, 53)...)
+	msg = append(msg, tupleAttr(ctaTupleReply, "198.51.100.9", "203.0.113.10", 17, 53, 5353)...)
+	if c, _ := parse(msg); c.SNATTo != "203.0.113.10" || c.DNATTo != "" || c.Protocol != "udp" {
+		t.Errorf("masqueraded: %+v", c)
+	}
+}

internal/engine/engine.go

@@ -0,0 +1,642 @@
+// Package engine changes Housecarl's configuration. Every change, from the
+// API, the CLI or an import, goes through Apply: the change is made on a
+// copy of the state, the result is validated as a whole, the ruleset built
+// from it is checked by nftables and put in force, and only then are the
+// changed objects and a history entry saved together (if saving fails, the
+// previous ruleset is put back). Revert undoes a history entry the same
+// way.
+package engine
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"housecarl/internal/auth"
+	"housecarl/internal/model"
+	"housecarl/internal/store"
+	"housecarl/internal/validate"
+)
+
+// Kinds of errors, for the API's status codes.
+const (
+	KindBadRequest = "bad_request"
+	KindNotFound   = "not_found"
+	KindConflict   = "conflict"
+	KindValidation = "validation"
+	// KindFirewall: nftables refused or failed to apply the ruleset.
+	KindFirewall = "firewall"
+)
+
+// Firewall puts configurations in force (internal/nft, through
+// internal/render).
+type Firewall interface {
+	// Check builds the ruleset and has nftables check it, changing
+	// nothing.
+	Check(ctx context.Context, st *model.State) error
+	// Apply puts the configuration's ruleset in force, in one
+	// transaction.
+	Apply(ctx context.Context, st *model.State) error
+}
+
+// Error is a refused change.
+type Error struct {
+	Kind     string             `json:"code"`
+	Message  string             `json:"message"`
+	Problems []validate.Problem `json:"problems,omitempty"`
+}
+
+func (e *Error) Error() string { return e.Message }
+
+// Errorf makes an Error.
+func Errorf(kind, format string, args ...any) *Error {
+	return &Error{Kind: kind, Message: fmt.Sprintf(format, args...)}
+}
+
+// AsError unwraps an Error.
+func AsError(err error) (*Error, bool) {
+	var e *Error
+	ok := errors.As(err, &e)
+	return e, ok
+}
+
+// Result describes an applied (or previewed) change.
+type Result struct {
+	Applied   bool               `json:"applied"`
+	DryRun    bool               `json:"dry_run,omitempty"`
+	HistoryID int64              `json:"history_id,omitempty"`
+	Changes   []model.Change     `json:"changes"`
+	Warnings  []validate.Problem `json:"warnings,omitempty"`
+	// Before and After are the configurations the change goes from and
+	// to (After also for a dry run), for diffs and answers.
+	Before, After *model.State `json:"-"`
+}
+
+// Engine holds the current configuration.
+type Engine struct {
+	store *store.Store
+	log   *slog.Logger
+	// Env describes the host's interfaces for validation (nil: skip).
+	Env func() *validate.Env
+	// Firewall puts each new state in force (nil: none, in tests that
+	// only look at the configuration).
+	Firewall Firewall
+	// OnChange is called with each new state, after it was saved.
+	OnChange func(*model.State)
+
+	mu    sync.Mutex
+	state *model.State
+}
+
+// New loads the configuration.
+func New(ctx context.Context, st *store.Store, log *slog.Logger) (*Engine, error) {
+	if log == nil {
+		log = slog.Default()
+	}
+	s, err := st.LoadState(ctx)
+	if err != nil {
+		return nil, err
+	}
+	Normalize(s)
+	return &Engine{store: st, log: log, state: s}, nil
+}
+
+// Store is the engine's database.
+func (e *Engine) Store() *store.Store { return e.store }
+
+// State returns a copy of the current configuration.
+func (e *Engine) State() *model.State {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.state.Clone()
+}
+
+func (e *Engine) env() *validate.Env {
+	if e.Env == nil {
+		return nil
+	}
+	return e.Env()
+}
+
+// Problems validates the current configuration (for the status page).
+func (e *Engine) Problems() []validate.Problem {
+	return validate.Check(e.State(), e.env())
+}
+
+// Edit changes a configuration in place. NewID gives IDs for new objects.
+type Edit func(st *model.State, newID func() int64) error
+
+// Options say how to apply a change.
+type Options struct {
+	DryRun bool
+	Via    string // web, cli, api, import
+	// RevertOf is the history entry the change undoes (a changeset with a
+	// staged undo), so History links the two.
+	RevertOf int64
+}
+
+// Apply makes a change: edit runs on a copy of the configuration, the
+// result is validated, and the changed objects are saved with a history
+// entry. A change that touches nothing is not recorded.
+func (e *Engine) Apply(ctx context.Context, p *auth.Principal, description string, opts Options, edit Edit) (*Result, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	next := e.state.Clone()
+	var idErr error
+	newID := func() int64 {
+		ids, err := e.store.AllocateIDs(ctx, 1)
+		if err != nil {
+			idErr = err
+			return 0
+		}
+		return ids[0]
+	}
+	if err := edit(next, newID); err != nil {
+		return nil, err
+	}
+	if idErr != nil {
+		return nil, idErr
+	}
+	return e.commit(ctx, p, description, opts, next, opts.RevertOf)
+}
+
+// commit validates next and saves it as the new state.
+func (e *Engine) commit(ctx context.Context, p *auth.Principal, description string, opts Options, next *model.State, revertOf int64) (*Result, error) {
+	Normalize(next)
+	next.Sort()
+	changes := Diff(e.state, next)
+	res := &Result{DryRun: opts.DryRun, Changes: changes, Before: e.state.Clone(), After: next.Clone()}
+	if len(changes) == 0 {
+		return res, nil
+	}
+	problems := validate.Check(next, e.env())
+	if validate.HasErrors(problems) {
+		errs := validate.Errors(problems)
+		msg := errs[0].Message
+		if errs[0].Object != "" {
+			msg = errs[0].Object + ": " + msg
+		}
+		if len(errs) > 1 {
+			msg += fmt.Sprintf(" (and %d more problems)", len(errs)-1)
+		}
+		return res, &Error{Kind: KindValidation, Message: msg, Problems: errs}
+	}
+	// Only report warnings the change brings; old ones are on the status
+	// page.
+	res.Warnings = newProblems(validate.Check(e.state, e.env()), problems)
+	res.Warnings = append(res.Warnings, quotaRestarts(e.state, next)...)
+	if e.Firewall != nil {
+		if err := e.Firewall.Check(ctx, next); err != nil {
+			return res, &Error{Kind: KindFirewall, Message: "nftables refused the ruleset: " + err.Error()}
+		}
+	}
+	if opts.DryRun {
+		return res, nil
+	}
+	if e.Firewall != nil {
+		if err := e.Firewall.Apply(ctx, next); err != nil {
+			return res, &Error{Kind: KindFirewall, Message: "applying the ruleset failed, nothing was changed: " + err.Error()}
+		}
+	}
+	entry := &model.HistoryEntry{Time: time.Now(), Username: p.Name(), Via: opts.Via, Description: description, Changes: changes, RevertOf: revertOf}
+	if p != nil && p.User != nil {
+		entry.UserID = p.User.ID
+	}
+	if entry.Description == "" {
+		entry.Description = Describe(changes)
+	}
+	if err := e.store.SaveChanges(ctx, changes, entry); err != nil {
+		// The new ruleset is in force but not saved: put the old one back.
+		if e.Firewall != nil {
+			if rerr := e.Firewall.Apply(ctx, e.state); rerr != nil {
+				e.log.Error("saving a change failed, and putting the previous ruleset back failed too", "err", rerr)
+			}
+		}
+		return res, err
+	}
+	e.state = next
+	res.Applied, res.HistoryID = true, entry.ID
+	e.Audit(ctx, p, "config.change", fmt.Sprintf("history %d", entry.ID), "success", entry.Description)
+	e.log.Info("configuration changed", "history", entry.ID, "user", p.Name(), "via", opts.Via, "description", entry.Description)
+	if e.OnChange != nil {
+		e.OnChange(next.Clone())
+	}
+	return res, nil
+}
+
+// quotaRestarts warns of the rules whose quota starts again because they
+// now count it in a way what they used can't be carried to.
+func quotaRestarts(before, after *model.State) []validate.Problem {
+	was := map[int64]*model.Rule{}
+	for _, r := range before.Rules {
+		was[r.ID] = r
+	}
+	var out []validate.Problem
+	for _, r := range after.Rules {
+		if carry, why := model.QuotaCarry(was[r.ID], r); carry == model.QuotaRestarts {
+			out = append(out, validate.Problem{Severity: validate.Warning, Code: "quota_restarts", Object: r.Label(),
+				Message: "its quota starts again: " + why})
+		}
+	}
+	return out
+}
+
+func newProblems(before, after []validate.Problem) []validate.Problem {
+	seen := map[string]bool{}
+	for _, p := range before {
+		seen[p.Code+"\x00"+p.Object+"\x00"+p.Message] = true
+	}
+	var out []validate.Problem
+	for _, p := range after {
+		if !seen[p.Code+"\x00"+p.Object+"\x00"+p.Message] {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+// ApplyCurrent puts the current configuration in force again: at start,
+// and when a flowtable device appears. It holds the engine's lock, so a
+// change can't be applied in between.
+func (e *Engine) ApplyCurrent(ctx context.Context) error {
+	if e.Firewall == nil {
+		return nil
+	}
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.Firewall.Apply(ctx, e.state.Clone())
+}
+
+// Revert undoes a history entry, if the objects it changed are still as
+// it left them.
+func (e *Engine) Revert(ctx context.Context, p *auth.Principal, id int64, opts Options) (*Result, error) {
+	h, err := e.store.GetHistory(ctx, id)
+	if errors.Is(err, store.ErrNotFound) {
+		return nil, Errorf(KindNotFound, "history entry %d not found", id)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if h.RevertedBy != 0 {
+		return nil, Errorf(KindConflict, "history entry %d was already undone by entry %d", id, h.RevertedBy)
+	}
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	next := e.state.Clone()
+	if err := Undo(next, h); err != nil {
+		return nil, err
+	}
+	return e.commit(ctx, p, UndoDescription(h), opts, next, h.ID)
+}
+
+// UndoDescription is an undo's description: "undo 12: create rule …".
+func UndoDescription(h *model.HistoryEntry) string {
+	return fmt.Sprintf("undo %d: %s", h.ID, h.Description)
+}
+
+// Undo applies a history entry backwards to st, if every object it
+// changed is still as it left it (a change made since would be lost).
+func Undo(st *model.State, h *model.HistoryEntry) error {
+	cur := objects(st)
+	for i := len(h.Changes) - 1; i >= 0; i-- {
+		c := h.Changes[i]
+		now := cur[key{c.Kind, c.ID}]
+		if !sameJSON(now, c.After) {
+			what := c.Label
+			if what == "" {
+				what = fmt.Sprintf("%s %d", c.Kind, c.ID)
+			}
+			return Errorf(KindConflict, "%s %s has changed since entry %d; undo the later changes first, or change it by hand", c.Kind, what, h.ID)
+		}
+		if err := put(st, c.Kind, c.ID, c.Before); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Rewind applies a history entry backwards to st without Undo's check:
+// stepping back through the entries newest first brings back the
+// configuration as it was before each.
+func Rewind(st *model.State, h *model.HistoryEntry) error {
+	for i := len(h.Changes) - 1; i >= 0; i-- {
+		c := h.Changes[i]
+		if c.Kind == model.KindSettings && c.Before == nil {
+			continue
+		}
+		if err := put(st, c.Kind, c.ID, c.Before); err != nil {
+			return err
+		}
+	}
+	// put appends what it brings back; its position puts it in place.
+	st.Sort()
+	return nil
+}
+
+// Replace makes the configuration next as a whole (imports).
+func (e *Engine) Replace(ctx context.Context, p *auth.Principal, description string, opts Options, next *model.State) (*Result, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.commit(ctx, p, description, opts, next.Clone(), 0)
+}
+
+// Audit records an action in the audit log.
+func (e *Engine) Audit(ctx context.Context, p *auth.Principal, action, object, result, detail string) {
+	ev := &model.AuditEvent{Action: action, Object: object, Result: result, Detail: detail}
+	if p != nil {
+		ev.Username, ev.Auth, ev.IP = p.Name(), p.Method, p.IP
+		if p.User != nil {
+			ev.UserID = p.User.ID
+		}
+		if p.Token != nil {
+			ev.TokenID = p.Token.ID
+		}
+	}
+	if err := e.store.AddAudit(ctx, ev); err != nil {
+		e.log.Warn("writing the audit log failed", "err", err)
+	}
+}
+
+// ---- diffs
+
+type key struct {
+	kind string
+	id   int64
+}
+
+func objects(st *model.State) map[key]json.RawMessage {
+	m := map[key]json.RawMessage{{model.KindSettings, 0}: mustJSON(st.Settings)}
+	for _, k := range model.Kinds {
+		for _, o := range st.Objects(k) {
+			m[key{k, o.GetID()}] = mustJSON(o)
+		}
+	}
+	return m
+}
+
+func labels(st *model.State) map[key]string {
+	m := map[key]string{{model.KindSettings, 0}: "settings"}
+	for _, k := range model.Kinds {
+		for _, o := range st.Objects(k) {
+			m[key{k, o.GetID()}] = o.Label()
+		}
+	}
+	return m
+}
+
+// Diff lists what changes from a to b: settings, then each kind of
+// object (zones, sets, services, rules, NAT), each by ID.
+func Diff(a, b *model.State) []model.Change {
+	oa, ob := objects(a), objects(b)
+	la, lb := labels(a), labels(b)
+	var keys []key
+	for k := range oa {
+		keys = append(keys, k)
+	}
+	for k := range ob {
+		if _, ok := oa[k]; !ok {
+			keys = append(keys, k)
+		}
+	}
+	order := map[string]int{model.KindSettings: 0}
+	for i, k := range model.Kinds {
+		order[k] = i + 1
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		if keys[i].kind != keys[j].kind {
+			return order[keys[i].kind] < order[keys[j].kind]
+		}
+		return keys[i].id < keys[j].id
+	})
+	var out []model.Change
+	for _, k := range keys {
+		before, after := oa[k], ob[k]
+		if sameJSON(before, after) {
+			continue
+		}
+		label := lb[k]
+		if label == "" {
+			label = la[k]
+		}
+		out = append(out, model.Change{Kind: k.kind, ID: k.id, Label: label, Before: before, After: after})
+	}
+	return out
+}
+
+// Describe sums up changes: "create zone lan (eth1)", or
+// "3 changes" for many.
+func Describe(cs []model.Change) string {
+	if len(cs) == 0 {
+		return "no changes"
+	}
+	parts := []string{}
+	for _, c := range cs {
+		if c.Kind == model.KindSettings {
+			parts = append(parts, "update settings")
+			continue
+		}
+		parts = append(parts, c.Action()+" "+c.Kind+" "+c.Label)
+	}
+	if len(parts) > 3 {
+		return fmt.Sprintf("%s and %d more changes", strings.Join(parts[:2], ", "), len(parts)-2)
+	}
+	return strings.Join(parts, ", ")
+}
+
+func mustJSON(v any) json.RawMessage {
+	b, err := json.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}
+
+func sameJSON(a, b json.RawMessage) bool {
+	if a == nil || b == nil {
+		return a == nil && b == nil
+	}
+	return bytes.Equal(a, b)
+}
+
+// put sets (or with nil data deletes) one object in st.
+func put(st *model.State, kind string, id int64, data json.RawMessage) error {
+	if kind == model.KindSettings {
+		if data == nil {
+			return fmt.Errorf("settings can't be deleted")
+		}
+		var set model.Settings
+		if err := json.Unmarshal(data, &set); err != nil {
+			return err
+		}
+		st.Settings = set
+		return nil
+	}
+	return st.Put(kind, id, data)
+}
+
+// Normalize writes names, addresses, ports and lists in their canonical
+// form, so equal configurations compare equal, and spaces rule positions
+// out again when insertions have crowded them.
+func Normalize(st *model.State) {
+	st.Fill()
+	set := &st.Settings
+	set.AntispoofV4 = normAddrs(set.AntispoofV4)
+	set.AntispoofV6 = normAddrs(set.AntispoofV6)
+	set.OwnPrefixes = normAddrs(set.OwnPrefixes)
+	set.FlowtableDevices = normList(set.FlowtableDevices)
+	for _, z := range st.Zones {
+		z.Name = strings.TrimSpace(z.Name)
+		z.Interfaces = normList(z.Interfaces)
+	}
+	for _, x := range st.Sets {
+		x.Name = strings.TrimSpace(x.Name)
+		x.Entries = normAddrs(x.Entries)
+		if j := x.Jarl; j != nil {
+			for i, n := range j.Names {
+				j.Names[i] = model.ParseJarlName(n.Name)
+				if n.View != "" {
+					j.Names[i].View = strings.TrimSpace(n.View)
+				}
+			}
+			j.Within = normAddrs(j.Within) // zone names stay as they are
+		}
+	}
+	for _, x := range st.Services {
+		x.Name = strings.TrimSpace(x.Name)
+		x.Entries = normServices(x.Entries)
+	}
+	for _, r := range st.Rules {
+		normMatch(&r.Match)
+		r.CtState = normList(lower(r.CtState))
+		r.CtStatus = normList(lower(r.CtStatus))
+		r.Raw = strings.TrimSpace(r.Raw)
+	}
+	for _, f := range st.Forwards {
+		f.Protocols = normList(lower(f.Protocols))
+		f.PublicAddresses = normAddrs(f.PublicAddresses)
+		f.Sources = normAddrs(f.Sources)
+		f.HairpinSources = normAddrs(f.HairpinSources)
+		f.Ports = normPorts(f.Ports)
+		f.ToPort = normPorts(f.ToPort)
+		f.To = normAddr(f.To)
+		f.To6 = normAddr(f.To6)
+	}
+	for _, x := range st.SNATs {
+		normMatch(&x.Match)
+		x.To = normAddr(x.To)
+	}
+	for _, x := range st.Redirects {
+		normMatch(&x.Match)
+		x.ToPort = normPorts(x.ToPort)
+	}
+	for _, b := range st.BanLists {
+		b.Name = strings.TrimSpace(b.Name)
+		b.Scope = strings.ToLower(strings.TrimSpace(b.Scope))
+		b.Hosts = normAddrs(b.Hosts)
+		b.Never = normAddrs(b.Never)
+	}
+	// Positions are fractions after insertions between rules; renumber
+	// only when two have come too close to put another between.
+	st.Sort()
+	crowded := func(pos []float64) bool {
+		for i := 1; i < len(pos); i++ {
+			if pos[i]-pos[i-1] < 1e-6 {
+				return true
+			}
+		}
+		return false
+	}
+	byChain := map[string][]float64{}
+	for _, r := range st.Rules {
+		byChain[r.Chain] = append(byChain[r.Chain], r.Position)
+	}
+	re := false
+	for _, pos := range byChain {
+		re = re || crowded(pos)
+	}
+	var fp, sp, rp []float64
+	for _, x := range st.Forwards {
+		fp = append(fp, x.Position)
+	}
+	for _, x := range st.SNATs {
+		sp = append(sp, x.Position)
+	}
+	for _, x := range st.Redirects {
+		rp = append(rp, x.Position)
+	}
+	if re || crowded(fp) || crowded(sp) || crowded(rp) {
+		st.Renumber()
+	}
+}
+
+func normMatch(m *model.Match) {
+	m.InZones = normList(m.InZones)
+	m.OutZones = normList(m.OutZones)
+	m.Sources = normAddrs(m.Sources)
+	m.Destinations = normAddrs(m.Destinations)
+	m.Services = normServices(m.Services)
+	m.Family = strings.ToLower(strings.TrimSpace(m.Family))
+}
+
+// normList trims, drops empty entries and duplicates, keeping the order.
+func normList(in []string) []string {
+	var out []string
+	seen := map[string]bool{}
+	for _, s := range in {
+		s = strings.TrimSpace(s)
+		if s != "" && !seen[s] {
+			seen[s] = true
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+func lower(in []string) []string {
+	out := make([]string, len(in))
+	for i, s := range in {
+		out[i] = strings.ToLower(s)
+	}
+	return out
+}
+
+// normAddrs writes addresses canonically; set names stay as they are.
+func normAddrs(in []string) []string {
+	out := normList(in)
+	for i, s := range out {
+		out[i] = normAddr(s)
+	}
+	return normList(out)
+}
+
+func normAddr(s string) string {
+	s = strings.TrimSpace(s)
+	if n, err := model.NormalizeAddress(s); err == nil {
+		return n
+	}
+	return s
+}
+
+func normServices(in []string) []string {
+	out := normList(in)
+	for i, s := range out {
+		if e, err := model.ParseServiceEntry(s); err == nil {
+			out[i] = e.String()
+		}
+	}
+	return normList(out)
+}
+
+func normPorts(s string) string {
+	if n, err := model.NormalizePorts(s); err == nil {
+		return n
+	}
+	return strings.TrimSpace(s)
+}

internal/engine/engine_test.go

@@ -0,0 +1,221 @@
+package engine
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"housecarl/internal/auth"
+	"housecarl/internal/model"
+	"housecarl/internal/store"
+)
+
+func newEngine(t *testing.T) *Engine {
+	t.Helper()
+	st, err := store.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	e, err := New(context.Background(), st, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return e
+}
+
+var admin = &auth.Principal{User: &model.User{ID: 1, Username: "admin", Admin: true}, Method: "session"}
+
+// fakeFirewall records what it was asked to apply.
+type fakeFirewall struct {
+	applied  []*model.State
+	checkErr error
+	applyErr error
+}
+
+func (f *fakeFirewall) Check(_ context.Context, st *model.State) error { return f.checkErr }
+func (f *fakeFirewall) Apply(_ context.Context, st *model.State) error {
+	if f.applyErr != nil {
+		return f.applyErr
+	}
+	f.applied = append(f.applied, st.Clone())
+	return nil
+}
+
+func addWAN(t *testing.T, e *Engine) int64 {
+	t.Helper()
+	var id int64
+	res, err := e.Apply(context.Background(), admin, "", Options{Via: "cli"}, func(st *model.State, newID func() int64) error {
+		id = newID()
+		st.Zones = append(st.Zones, &model.Zone{ID: id, Name: "wan", Interfaces: []string{" wan ", "wan", "ppp*"}, External: true})
+		return nil
+	})
+	if err != nil || !res.Applied || res.HistoryID == 0 {
+		t.Fatalf("apply: %+v %v", res, err)
+	}
+	return id
+}
+
+func TestApplyNormalizesAndRecords(t *testing.T) {
+	e := newEngine(t)
+	fw := &fakeFirewall{}
+	e.Firewall = fw
+	id := addWAN(t, e)
+	z := e.State().ByID(model.KindZone, id).(*model.Zone)
+	if strings.Join(z.Interfaces, ",") != "wan,ppp*" {
+		t.Errorf("interfaces not normalized: %q", z.Interfaces)
+	}
+	if len(fw.applied) != 1 || len(fw.applied[0].Zones) != 1 {
+		t.Errorf("firewall got %+v", fw.applied)
+	}
+	h, _ := e.Store().ListHistory(context.Background(), 10, 0)
+	if len(h) != 1 || h[0].Description != "create zone wan" || h[0].Via != "cli" || h[0].Username != "admin" {
+		t.Errorf("history %+v", h[0])
+	}
+	// Nothing changed: nothing recorded or applied.
+	res, err := e.Apply(context.Background(), admin, "", Options{}, func(st *model.State, _ func() int64) error { return nil })
+	if err != nil || res.Applied || len(res.Changes) != 0 || len(fw.applied) != 1 {
+		t.Errorf("empty change: %+v %v", res, err)
+	}
+}
+
+func TestValidationAndFirewallRefusals(t *testing.T) {
+	e := newEngine(t)
+	fw := &fakeFirewall{}
+	e.Firewall = fw
+	addWAN(t, e)
+	addRule := func(r *model.Rule) (*Result, error) {
+		return e.Apply(context.Background(), admin, "", Options{}, func(st *model.State, newID func() int64) error {
+			r.ID = newID()
+			st.Rules = append(st.Rules, r)
+			return nil
+		})
+	}
+	// Validation: an unknown set.
+	_, err := addRule(&model.Rule{Chain: model.ChainInput, Position: 1, Action: model.ActionAccept, Match: model.Match{Sources: []string{"trusted"}}})
+	if ee, ok := AsError(err); !ok || ee.Kind != KindValidation || ee.Problems[0].Code != "unknown_set" {
+		t.Errorf("unknown set: %v", err)
+	}
+	// nftables refuses: nothing applied or saved.
+	fw.checkErr = errors.New("syntax error")
+	_, err = addRule(&model.Rule{Chain: model.ChainInput, Position: 1, Action: model.ActionAccept, Raw: "bogus"})
+	if ee, ok := AsError(err); !ok || ee.Kind != KindFirewall {
+		t.Errorf("refused ruleset: %v", err)
+	}
+	// Applying fails: nothing saved.
+	fw.checkErr, fw.applyErr = nil, errors.New("netlink: busy")
+	_, err = addRule(&model.Rule{Chain: model.ChainInput, Position: 1, Action: model.ActionAccept, Match: model.Match{Services: []string{"ssh"}}})
+	if ee, ok := AsError(err); !ok || ee.Kind != KindFirewall || !strings.Contains(ee.Message, "nothing was changed") {
+		t.Errorf("failed apply: %v", err)
+	}
+	if len(e.State().Rules) != 0 {
+		t.Errorf("rules saved after failures: %+v", e.State().Rules)
+	}
+	// A dry run is checked, but neither applied nor saved.
+	fw.applyErr = nil
+	before := len(fw.applied)
+	res, err := e.Apply(context.Background(), admin, "", Options{DryRun: true}, func(st *model.State, newID func() int64) error {
+		st.Rules = append(st.Rules, &model.Rule{ID: newID(), Chain: model.ChainInput, Position: 1, Action: model.ActionDrop})
+		return nil
+	})
+	if err != nil || res.Applied || len(res.Changes) != 1 || len(fw.applied) != before {
+		t.Errorf("dry run: %+v %v", res, err)
+	}
+}
+
+func TestRevertAppliesThePreviousRuleset(t *testing.T) {
+	e := newEngine(t)
+	fw := &fakeFirewall{}
+	e.Firewall = fw
+	addWAN(t, e)
+	res, err := e.Apply(context.Background(), admin, "", Options{}, func(st *model.State, newID func() int64) error {
+		st.Settings.ForwardPolicy = model.PolicyAccept
+		return nil
+	})
+	if err != nil || !res.Applied {
+		t.Fatal(err)
+	}
+	if _, err := e.Revert(context.Background(), admin, res.HistoryID, Options{}); err != nil {
+		t.Fatal(err)
+	}
+	last := fw.applied[len(fw.applied)-1]
+	if last.Settings.ForwardPolicy != model.PolicyDrop || e.State().Settings.ForwardPolicy != model.PolicyDrop {
+		t.Errorf("after revert: applied %s, state %s", last.Settings.ForwardPolicy, e.State().Settings.ForwardPolicy)
+	}
+	// Undoing it twice is refused.
+	if _, err := e.Revert(context.Background(), admin, res.HistoryID, Options{}); err == nil {
+		t.Error("second undo accepted")
+	}
+}
+
+func TestPositionsStayStableUntilCrowded(t *testing.T) {
+	e := newEngine(t)
+	for i := 0; i < 3; i++ {
+		e.Apply(context.Background(), admin, "", Options{}, func(st *model.State, newID func() int64) error {
+			st.Rules = append(st.Rules, &model.Rule{ID: newID(), Chain: model.ChainInput, Position: float64(i + 1), Action: model.ActionAccept,
+				Match: model.Match{Services: []string{"ssh"}}})
+			return nil
+		})
+	}
+	// Inserting between two rules changes only the new one.
+	res, err := e.Apply(context.Background(), admin, "", Options{}, func(st *model.State, newID func() int64) error {
+		st.Rules = append(st.Rules, &model.Rule{ID: newID(), Chain: model.ChainInput, Position: 1.5, Action: model.ActionDrop})
+		return nil
+	})
+	if err != nil || len(res.Changes) != 1 {
+		t.Fatalf("insert: %+v %v", res, err)
+	}
+	var order []string
+	for _, r := range e.State().Rules {
+		order = append(order, r.Action)
+	}
+	if strings.Join(order, ",") != "accept,drop,accept,accept" {
+		t.Errorf("order %v", order)
+	}
+}
+
+// TestQuotaRestartsWarns: a change that counts a quota so what it used
+// can't carry over warns; one that can (to a total, wider groups) doesn't.
+func TestQuotaRestartsWarns(t *testing.T) {
+	e := newEngine(t)
+	ctx := context.Background()
+	var id int64
+	if _, err := e.Apply(ctx, admin, "", Options{}, func(st *model.State, newID func() int64) error {
+		id = newID()
+		st.Rules = append(st.Rules, &model.Rule{ID: id, Chain: "forward", Action: "drop", Quota: &model.Quota{Bytes: 1 << 30, Over: true, PerSource: true}})
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	change := func(f func(r *model.Rule)) []string {
+		res, err := e.Apply(ctx, admin, "", Options{DryRun: true}, func(st *model.State, _ func() int64) error {
+			f(st.ByID(model.KindRule, id).(*model.Rule))
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var codes []string
+		for _, w := range res.Warnings {
+			if w.Code == "quota_restarts" {
+				codes = append(codes, w.Message)
+			}
+		}
+		return codes
+	}
+	if w := change(func(r *model.Rule) { r.Quota.PerSource, r.Quota.PerDestination = false, true }); len(w) != 1 || !strings.Contains(w[0], "per source can't be counted per destination") {
+		t.Errorf("per source to per destination: %v", w)
+	}
+	if w := change(func(r *model.Rule) { r.Quota.PerSource = false }); len(w) != 0 {
+		t.Errorf("to a total (summed): %v", w)
+	}
+	if w := change(func(r *model.Rule) { r.GroupV6 = 56 }); len(w) != 0 {
+		t.Errorf("wider groups: %v", w)
+	}
+	if w := change(func(r *model.Rule) { r.GroupV6 = 128 }); len(w) != 1 {
+		t.Errorf("finer groups: %v", w)
+	}
+	if w := change(func(r *model.Rule) { r.Quota.Bytes = 2 << 30 }); len(w) != 0 {
+		t.Errorf("another amount: %v", w)
+	}
+}

internal/hostif/hostif.go

@@ -0,0 +1,87 @@
+// Package hostif reads the machine's network interfaces: their IPv4 and
+// IPv6 addresses and whether they are up. Results are cached for a few
+// seconds, since validation and the status page ask often.
+package hostif
+
+import (
+	"net"
+	"net/netip"
+	"sync"
+	"time"
+)
+
+// Host reads this machine's interfaces.
+type Host struct {
+	mu    sync.Mutex
+	at    time.Time
+	addrs map[string][]netip.Prefix
+	up    map[string]bool
+	err   error
+}
+
+func (h *Host) refresh() {
+	if time.Since(h.at) < 5*time.Second && (h.addrs != nil || h.err != nil) {
+		return
+	}
+	h.at = time.Now()
+	ifs, err := net.Interfaces()
+	if err != nil {
+		h.addrs, h.up, h.err = nil, nil, err
+		return
+	}
+	addrs, up := map[string][]netip.Prefix{}, map[string]bool{}
+	for _, i := range ifs {
+		var ps []netip.Prefix
+		as, _ := i.Addrs()
+		for _, a := range as {
+			if n, ok := a.(*net.IPNet); ok {
+				if ip, ok := netip.AddrFromSlice(n.IP); ok {
+					bits, _ := n.Mask.Size()
+					ps = append(ps, netip.PrefixFrom(ip.Unmap(), bits))
+				}
+			}
+		}
+		addrs[i.Name] = ps
+		up[i.Name] = i.Flags&net.FlagUp != 0
+	}
+	h.addrs, h.up, h.err = addrs, up, nil
+}
+
+// All returns every interface with its addresses.
+func (h *Host) All() (map[string][]netip.Prefix, error) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.refresh()
+	return h.addrs, h.err
+}
+
+// Up reports whether an interface is up.
+func (h *Host) Up(name string) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.refresh()
+	return h.up[name]
+}
+
+// Forget drops what was read, so the next question reads the interfaces
+// again (after the kernel said one appeared or went).
+func (h *Host) Forget() {
+	h.mu.Lock()
+	h.at = time.Time{}
+	h.mu.Unlock()
+}
+
+// Exists returns the set of interface names (nil if they can't be read).
+func (h *Host) Exists() map[string]bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.refresh()
+	if h.err != nil {
+		return nil
+	}
+	out := make(map[string]bool, len(h.addrs))
+	for n := range h.addrs {
+		out[n] = true
+	}
+	return out
+}

internal/importer/audit_test.go

@@ -0,0 +1,98 @@
+package importer
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"housecarl/internal/render"
+	"housecarl/internal/validate"
+)
+
+// TestImportAudit: rulesets the importer must not change the meaning of
+// (testdata/audit.nft, as nft 1.1.3 lists it).
+func TestImportAudit(t *testing.T) {
+	js, _ := os.ReadFile("testdata/audit.json")
+	txt, _ := os.ReadFile("testdata/audit.txt")
+	defs, _ := os.ReadFile("testdata/audit.nft")
+	res, err := Import(Input{JSON: js, Text: string(txt), Defines: string(defs), Zones: map[string]string{"enp2s0.20": "iot", "eth9": "x"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	st := res.State
+	notes := strings.Join(res.Notes, "\n")
+	text := render.Render(st, render.Options{}).Text
+	has := func(what, s string) {
+		t.Helper()
+		if !strings.Contains(text, s) {
+			t.Errorf("%s: no %q in\n%s", what, s, text)
+		}
+	}
+	// Two chains on the input hook: their rules once each, in priority
+	// order; established stays a rule after the first chain's rule.
+	if n := strings.Count(text, "192.0.2.66"); n != 1 {
+		t.Errorf("the early input chain's rule %d times", n)
+	}
+	if strings.Index(text, "192.0.2.66") > strings.Index(text, "ct state established,related counter accept comment \"hc:r") ||
+		!strings.Contains(notes, "both hook input") {
+		t.Errorf("chains on one hook:\n%s", text)
+	}
+	// A second match on a field, and != on the hop limit: raw, as written.
+	has("two source matches", "ip saddr 10.0.0.0/8 ip saddr != 10.0.10.0/24 tcp dport 22 drop")
+	has("hop limit", "ip6 hoplimit != 255 drop")
+	// A jump: a rule within it is the rule's own; one outside it, or raw,
+	// keeps the jump's match in front.
+	has("within the jump", `ip saddr 10.9.0.5 tcp dport 22 counter accept comment "hc:r`)
+	has("outside the jump", "ip saddr 10.9.0.0/24 ip saddr 10.8.0.1 tcp dport 23 accept")
+	has("raw under the jump", "ip saddr 10.9.0.0/24 meta mark 0x00000001 accept")
+	// Sets: a timeout set is dynamic; an interface set is written out in
+	// the raw rule that names it.
+	if s := st.Set("blocklist"); s == nil || !s.Dynamic || s.Timeout != 3600 {
+		t.Errorf("blocklist: %+v", s)
+	}
+	has("dynamic set", "ip saddr @blocklist_v4 counter drop")
+	has("interface set", `iifname { "enp2s0", "enp3s0" } accept`)
+	// An accept to a forwarded host that isn't only for forwarded
+	// connections stays a rule; a DNAT of two ports, two forwards.
+	has("not only forwarded", `ip daddr 10.0.10.37 tcp dport 443 counter accept comment "hc:r`)
+	if len(st.Forwards) != 2 || st.Forwards[0].Name != "web-80" || st.Forwards[1].Name != "web-443" {
+		t.Errorf("forwards %+v", st.Forwards)
+	}
+	// Zones: as given (a VLAN), from defines, and why.
+	if z := st.Zone("iot"); z == nil || z.Interfaces[0] != "enp2s0.20" {
+		t.Errorf("zone iot: %+v", st.Zones)
+	}
+	if st.Zone("lan") == nil || st.Zone("wan") == nil {
+		t.Errorf("zones %+v", st.Zones)
+	}
+	for _, want := range []string{"iot (enp2s0.20: as given)", "lan (enp2s0: the define lan_if)", "--zone eth9=x: no rule names"} {
+		if !strings.Contains(notes, want) {
+			t.Errorf("no note %q in\n%s", want, notes)
+		}
+	}
+	if ps := validate.Check(st, nil); validate.HasErrors(ps) {
+		t.Errorf("problems: %+v", validate.Errors(ps))
+	}
+	// And nftables takes it.
+	cmd := strings.Fields(os.Getenv("HOUSECARL_NFT"))
+	if len(cmd) == 0 {
+		return
+	}
+	c := exec.CommandContext(context.Background(), cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...)
+	c.Stdin = strings.NewReader(text)
+	if out, err := c.CombinedOutput(); err != nil {
+		t.Errorf("nft -c: %v\n%s\n%s", err, out, text)
+	}
+}
+
+// TestValidIface: interface names a define may hold, VLANs too.
+func TestValidIface(t *testing.T) {
+	for s, want := range map[string]bool{"enp2s0": true, "wan": true, "enp2s0.20": true, "br0.4094": true, "veth*": true,
+		"10.0.10.5": false, "example.com": false, "a.b.c": false, "enp2s0.20.30": false, "averyveryverylongname": false} {
+		if validIface(s) != want {
+			t.Errorf("validIface(%q) = %v", s, !want)
+		}
+	}
+}

internal/importer/importer_test.go

@@ -0,0 +1,323 @@
+package importer
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"housecarl/internal/model"
+	"housecarl/internal/render"
+	"housecarl/internal/validate"
+)
+
+// The maintainer's router ruleset (testdata/router.nft), as nft 1.1.3
+// lists it in JSON (router.json) and text (router.txt).
+func importRouter(t *testing.T) *Result {
+	t.Helper()
+	js, _ := os.ReadFile("testdata/router.json")
+	txt, _ := os.ReadFile("testdata/router.txt")
+	defs, _ := os.ReadFile("testdata/router.nft")
+	res, err := Import(Input{JSON: js, Text: string(txt), Defines: string(defs)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return res
+}
+
+func TestImportRouter(t *testing.T) {
+	res := importRouter(t)
+	st := res.State
+	for _, n := range res.Notes {
+		t.Log("note:", n)
+	}
+	for _, r := range res.Raw {
+		t.Log("raw:", r)
+	}
+	// Zones, named by the defines; wan is external.
+	wan := st.Zone("wan")
+	if wan == nil || !wan.External || wan.Interfaces[0] != "wan" {
+		t.Errorf("zone wan: %+v", wan)
+	}
+	if z := st.Zone("dhcp"); z == nil || z.Interfaces[0] != "enp2s0" {
+		t.Errorf("zone dhcp: %+v (zones %+v)", z, st.Zones)
+	}
+	// Sets named by the defines.
+	if s := st.Set("trusted_ips"); s == nil || len(s.Entries) != 7 {
+		t.Errorf("set trusted_ips: %+v", s)
+	}
+	if s := st.Set("masq_to_internet"); s == nil {
+		t.Errorf("no set masq_to_internet (sets %v)", setNames(st))
+	}
+	// Built-ins: the ones at the head of their chains, up to the invalid
+	// drop, which stays a rule (below): the ICMP, ICMPv6 and DHCPv6
+	// accepts after it stay rules too, in their order, so nothing the
+	// drop stopped gets through ahead of it.
+	b := st.Settings.Builtins
+	if !b.Loopback || !b.Established || b.ICMPv6 || b.PingInternal || b.DHCPv6Client || !b.MSSClamp || !b.Antispoof {
+		t.Errorf("built-ins %+v", b)
+	}
+	var input []string
+	for _, r := range st.Rules {
+		if r.Chain == model.ChainInput {
+			input = append(input, strings.Join(r.CtState, ",")+"|"+strings.Join(r.Services, ",")+"|"+r.Raw)
+		}
+	}
+	if len(input) < 5 || input[0] != "invalid||" || !strings.HasPrefix(input[1], "|icmp/echo-reply,icmp/echo-request|") ||
+		!strings.Contains(input[2], "icmpv6/packet-too-big") || !strings.Contains(input[3], "ip6 hoplimit 255") || !strings.Contains(input[4], "udp/546") {
+		t.Errorf("input rules start %q", input)
+	}
+	// The forward chain's asymmetric-routing rule sits between the
+	// established and invalid rules: invalid stays rules, in order.
+	if b.DropInvalid {
+		t.Error("drop_invalid became a built-in, which would put it before the asymmetric-routing rule")
+	}
+	var fwd []string
+	for _, r := range st.Rules {
+		if r.Chain == model.ChainForward {
+			fwd = append(fwd, r.Comment+"|"+strings.Join(r.CtState, ","))
+		}
+	}
+	if len(fwd) < 2 || fwd[0] != "Needed due to asymmetric routing (internal only)|invalid" || fwd[1] != "|invalid" {
+		t.Errorf("forward rules start %q", fwd)
+	}
+	if st.Settings.OwnPrefixes[0] != "2a02:8013:7f3d::/48" || len(st.Settings.AntispoofV4) != 6 {
+		t.Errorf("antispoof %+v %+v", st.Settings.OwnPrefixes, st.Settings.AntispoofV4)
+	}
+	// Port forwards with their hairpin; the forward-chain accepts are
+	// folded in.
+	if len(st.Forwards) != 5 {
+		t.Fatalf("forwards %+v", st.Forwards)
+	}
+	jf := st.Forwards[0]
+	if jf.Name != "jellyfin" || jf.Ports != "8096" || jf.To != "10.0.10.57" || jf.ToPort != "443" || jf.PublicAddresses[0] != "217.155.8.194" ||
+		!jf.Hairpin || jf.HairpinSources[0] != "internal_net" {
+		t.Errorf("jellyfin %+v", jf)
+	}
+	for _, r := range st.Rules {
+		if r.Chain == model.ChainForward && strings.Contains(r.Comment, "Forward") && len(r.CtStatus) > 0 {
+			t.Errorf("forward accept not folded in: %+v", r)
+		}
+	}
+	// Source NAT: the masquerades.
+	if len(st.SNATs) != 2 || st.SNATs[0].OutZones[0] != "wan" || st.SNATs[0].Sources[0] != "masq_to_internet" ||
+		!st.SNATs[1].NotOut || st.SNATs[1].OutZones[0] != "wg0" {
+		t.Errorf("snats %+v %+v", st.SNATs[0], st.SNATs[1])
+	}
+	// The jump to wg0_input is inlined: its rules carry the source.
+	var wgDNS bool
+	for _, r := range st.Rules {
+		if r.Chain == model.ChainInput && len(r.Sources) == 1 && r.Sources[0] == "wg0_net" && strings.Join(r.Services, ",") == "udp/53" {
+			wgDNS = true
+		}
+	}
+	if !wgDNS {
+		t.Errorf("wg0_input not inlined")
+	}
+	// The jump's and the rule's sources both: the rule's lie within the
+	// jump's, so they say it all, without a note.
+	var wgSSH bool
+	for _, r := range st.Rules {
+		if r.Chain == model.ChainInput && strings.Join(r.Services, ",") == "tcp/22" && strings.Join(r.Sources, ",") == "10.50.50.3,10.50.50.6" {
+			wgSSH = true
+		}
+	}
+	if !wgSSH || strings.Contains(strings.Join(res.Notes, "\n"), "jump") {
+		t.Errorf("the WireGuard SSH rule: found %v, notes %q", wgSSH, res.Notes)
+	}
+	// The imported configuration is valid, and renders.
+	if ps := validate.Check(st, nil); validate.HasErrors(ps) {
+		t.Errorf("problems: %+v", validate.Errors(ps))
+	}
+	text := render.Render(st, render.Options{}).Text
+	for _, want := range []string{
+		`ip daddr 217.155.8.194 tcp dport 8096 counter dnat ip to 10.0.10.57:443`,
+		`ip saddr @internal_net_v4 ip daddr 10.0.10.57 tcp dport 443 ct status dnat counter masquerade`,
+		`oifname "wan" ip saddr @masq_to_internet_v4 counter masquerade`,
+		`oifname "wan" tcp flags syn tcp option maxseg size set rt mtu`,
+		`ip saddr @trusted_ips_v4 tcp dport 8053 counter accept comment "hc:r`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Errorf("render: missing %q", want)
+		}
+	}
+	if t.Failed() {
+		t.Log(text)
+	}
+}
+
+func setNames(st *model.State) []string {
+	var out []string
+	for _, s := range st.Sets {
+		out = append(out, s.Name)
+	}
+	return out
+}
+
+func TestImportNeedsATable(t *testing.T) {
+	js := []byte(`{"nftables":[{"table":{"family":"inet","name":"main"}},{"table":{"family":"ip","name":"nat"}}]}`)
+	if _, err := Import(Input{JSON: js}); err == nil || !strings.Contains(err.Error(), "several tables") {
+		t.Errorf("several tables: %v", err)
+	}
+	if _, err := Import(Input{JSON: js, Table: "ip nat"}); err != nil {
+		t.Errorf("--table: %v", err)
+	}
+}
+
+func TestForwardNames(t *testing.T) {
+	im := &importer{st: &model.State{}}
+	for _, c := range []string{"Port forward: Jellyfin", "8096 to media", "web", "web", "-"} {
+		im.st.Forwards = append(im.st.Forwards, &model.PortForward{Name: forwardName(c)})
+	}
+	im.nameForwards()
+	var got []string
+	for _, f := range im.st.Forwards {
+		got = append(got, f.Name)
+	}
+	if want := "jellyfin fwd-8096-to-media web web-2 "; strings.Join(got, " ") != want {
+		t.Errorf("names %q, want %q", strings.Join(got, " "), want)
+	}
+}
+
+// TestImportedRulesetPassesNft: the ruleset rendered from the import is
+// accepted by the real nft (HOUSECARL_NFT, as in internal/render).
+func TestImportedRulesetPassesNft(t *testing.T) {
+	cmd := strings.Fields(os.Getenv("HOUSECARL_NFT"))
+	if len(cmd) == 0 {
+		t.Skip("HOUSECARL_NFT not set")
+	}
+	rs := render.Render(importRouter(t).State, render.Options{})
+	c := exec.Command(cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...)
+	c.Stdin = strings.NewReader(rs.Text)
+	if out, err := c.CombinedOutput(); err != nil {
+		t.Errorf("nft -c: %v\n%s", err, out)
+	}
+}
+
+// TestImportMaps: dnat maps become port forwards and verdict maps rules,
+// one per element (testdata/maps.nft); a map that can't be expanded is
+// left out with a note, never kept as a raw rule naming it.
+func TestImportMaps(t *testing.T) {
+	js, _ := os.ReadFile("testdata/maps.json")
+	txt, _ := os.ReadFile("testdata/maps.txt")
+	res, err := Import(Input{JSON: js, Text: string(txt)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fwds []string
+	for _, f := range res.State.Forwards {
+		fwds = append(fwds, fmt.Sprintf("%s %s/%s->%s:%s", f.Name, strings.Join(f.Protocols, "+"), f.Ports, f.To, f.ToPort))
+	}
+	var rules []string
+	for _, r := range res.State.Rules {
+		rules = append(rules, fmt.Sprintf("%s in=%s from=%s %s %s%s", r.Chain, strings.Join(r.InZones, ","), strings.Join(r.Sources, ","),
+			strings.Join(r.Services, ","), r.Action, r.Raw))
+	}
+	want := []string{
+		"portforwards-80 tcp/80->192.168.1.50:8080",
+		"portforwards-443 tcp/443->192.168.1.50:",
+		"hosts-2222 tcp/2222->192.168.1.60:",
+		" udp/5000->192.168.1.70:5001",
+		" udp/6000->192.168.1.71:",
+	}
+	if got := strings.Join(fwds, "\n"); got != strings.Join(want, "\n") {
+		t.Errorf("forwards:\n%s\nwant:\n%s", got, strings.Join(want, "\n"))
+	}
+	// In order; the jump in the verdict map is inlined.
+	want = []string{
+		"input in= from=  accept",
+		"input in=lan from= tcp/22 accept",
+		"input in=lan from= tcp/23 drop",
+		"input in=lan from=192.168.1.0/24 tcp/80 accept",
+		"input in=wan from= udp/53 accept",
+		"input in=wan from= udp/123 drop",
+		"input in= from=10.0.0.0/8  accept",
+		"input in= from=192.0.2.1  drop",
+	}
+	if got := strings.Join(rules, "\n"); got != strings.Join(want, "\n") {
+		t.Errorf("rules:\n%s\nwant:\n%s", got, strings.Join(want, "\n"))
+	}
+	notes := strings.Join(res.Notes, "\n")
+	for _, n := range []string{"map portforwards: its 2 elements became 2 port forwards", "map hosts: its element became a port forward",
+		"uses the map @pairs, which couldn't be expanded"} {
+		if !strings.Contains(notes, n) {
+			t.Errorf("note %q missing:\n%s", n, notes)
+		}
+	}
+	if len(res.Raw) > 0 {
+		t.Errorf("raw rules: %v", res.Raw)
+	}
+	if ps := validate.Errors(validate.Check(res.State, nil)); len(ps) > 0 {
+		t.Errorf("invalid: %+v", ps)
+	}
+	if cmd := strings.Fields(os.Getenv("HOUSECARL_NFT")); len(cmd) > 0 {
+		c := exec.Command(cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...)
+		c.Stdin = strings.NewReader(render.Render(res.State, render.Options{}).Text)
+		if out, err := c.CombinedOutput(); err != nil {
+			t.Errorf("nft -c: %v\n%s", err, out)
+		}
+	}
+}
+
+// TestImportMeters: meters (per-address limits, connection counts and
+// quotas), byte rates and quotas become rules' limits and quotas
+// (testdata/meters.nft); the meters' sets aren't imported as address sets.
+func TestImportMeters(t *testing.T) {
+	js, _ := os.ReadFile("testdata/meters.json")
+	txt, _ := os.ReadFile("testdata/meters.txt")
+	res, err := Import(Input{JSON: js, Text: string(txt)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, r := range res.State.Rules {
+		s := r.Chain
+		if l := r.Limit; l != nil {
+			s += fmt.Sprintf(" limit %d %s/%s burst %d over=%v src=%v dst=%v", l.Rate, l.Unit, l.Per, l.Burst, l.Over, l.PerSource, l.PerDestination)
+		}
+		if r.ConnLimit > 0 {
+			s += fmt.Sprintf(" conn %d", r.ConnLimit)
+		}
+		if q := r.Quota; q != nil {
+			s += fmt.Sprintf(" quota %d over=%v src=%v", q.Bytes, q.Over, q.PerSource)
+		}
+		if r.GroupV4 != 0 || r.GroupV6 != 0 {
+			s += fmt.Sprintf(" group %d/%d", r.GroupV4, r.GroupV6)
+		}
+		s += " " + r.Action + r.Raw
+		got = append(got, s)
+	}
+	want := []string{
+		"input accept", // established: only in input here, so a rule
+		"input limit 3 /minute burst 5 over=true src=true dst=false drop",
+		"input limit 3 /minute burst 5 over=true src=true dst=false group 0/64 drop",
+		"input accept",
+		"forward conn 20 drop",
+		"forward limit 10240 kbytes/second burst 512 over=true src=false dst=false drop",
+		"forward quota 1048576000 over=true src=true group 24/0 drop",
+		"forward quota 104857600000 over=true src=false drop",
+		"forward accept",
+	}
+	if strings.Join(got, "\n") != strings.Join(want, "\n") {
+		t.Errorf("rules:\n%s\nwant:\n%s\nnotes: %v", strings.Join(got, "\n"), strings.Join(want, "\n"), res.Notes)
+	}
+	for _, s := range res.State.Sets {
+		if s.Name != "trusted" {
+			t.Errorf("a meter became a set: %s", s.Name)
+		}
+	}
+	if len(res.Raw) > 0 {
+		t.Errorf("raw rules: %v", res.Raw)
+	}
+	if ps := validate.Errors(validate.Check(res.State, nil)); len(ps) > 0 {
+		t.Errorf("invalid: %+v", ps)
+	}
+	if cmd := strings.Fields(os.Getenv("HOUSECARL_NFT")); len(cmd) > 0 {
+		c := exec.Command(cmd[0], append(cmd[1:], "-c", "-f", "/dev/stdin")...)
+		c.Stdin = strings.NewReader(render.Render(res.State, render.Options{}).Text)
+		if out, err := c.CombinedOutput(); err != nil {
+			t.Errorf("nft -c: %v\n%s", err, out)
+		}
+	}
+}

internal/importer/testdata/audit.json

@@ -0,0 +1 @@
+{"nftables": [{"metainfo": {"version": "1.1.3", "release_name": "Commodore Bullmoose #4", "json_schema_version": 1}}, {"table": {"family": "inet", "name": "audit", "handle": 1}}, {"chain": {"family": "inet", "table": "audit", "name": "early", "handle": 1, "type": "filter", "hook": "input", "prio": -10, "policy": "accept"}}, {"chain": {"family": "inet", "table": "audit", "name": "input", "handle": 2, "type": "filter", "hook": "input", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "audit", "name": "admin", "handle": 3}}, {"chain": {"family": "inet", "table": "audit", "name": "forward", "handle": 4, "type": "filter", "hook": "forward", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "audit", "name": "prerouting", "handle": 5, "type": "nat", "hook": "prerouting", "prio": -100, "policy": "accept"}}, {"set": {"family": "inet", "name": "blocklist", "table": "audit", "type": "ipv4_addr", "handle": 6, "flags": ["timeout"], "timeout": 3600}}, {"set": {"family": "inet", "name": "lans", "table": "audit", "type": "ifname", "handle": 7, "elem": ["enp2s0", "enp3s0"]}}, {"rule": {"family": "inet", "table": "audit", "chain": "early", "handle": 8, "comment": "an early input chain", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": "192.0.2.66"}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 9, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": ["established", "related"]}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 10, "comment": "dynamic", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": "@blocklist"}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 11, "comment": "an interface set", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "@lans"}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 12, "comment": "two source matches", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": {"prefix": {"addr": "10.0.0.0", "len": 8}}}}, {"match": {"op": "!=", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": {"prefix": {"addr": "10.0.10.0", "len": 24}}}}, {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 22}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 13, "comment": "hop limit not 255", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "icmpv6", "field": "type"}}, "right": "nd-neighbor-solicit"}}, {"match": {"op": "!=", "left": {"payload": {"protocol": "ip6", "field": "hoplimit"}}, "right": 255}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 14, "comment": "DNS from the IoT VLAN", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "enp2s0.20"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "udp", "field": "dport"}}, "right": 53}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "input", "handle": 15, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": {"prefix": {"addr": "10.9.0.0", "len": 24}}}}, {"jump": {"target": "admin"}}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "admin", "handle": 16, "comment": "within the jump", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": "10.9.0.5"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 22}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "admin", "handle": 17, "comment": "outside the jump", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": "10.8.0.1"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 23}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "admin", "handle": 18, "comment": "raw under the jump", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "mark"}}, "right": 1}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "forward", "handle": 19, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": ["established", "related"]}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "forward", "handle": 20, "comment": "not only forwarded", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "daddr"}}, "right": "10.0.10.37"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 443}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "audit", "chain": "forward", "hanLine truncated

internal/importer/testdata/audit.nft

@@ -0,0 +1,53 @@
+#! /sbin/nft -f
+# Cases the importer must not get wrong (importer_test.go TestImportAudit).
+
+define wan_if = "wan"
+define lan_if = "enp2s0"
+define iot_if = "enp2s0.20"
+define admins = { 10.9.0.0/24 }
+
+table inet audit {
+	set blocklist {
+		type ipv4_addr
+		flags timeout
+		timeout 1h
+	}
+	set lans {
+		type ifname
+		elements = { "enp2s0", "enp3s0" }
+	}
+
+	chain early {
+		type filter hook input priority -10; policy accept;
+		ip saddr 192.0.2.66 drop comment "an early input chain"
+	}
+
+	chain input {
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept
+		ip saddr @blocklist drop comment "dynamic"
+		iifname @lans accept comment "an interface set"
+		ip saddr 10.0.0.0/8 ip saddr != 10.0.10.0/24 tcp dport 22 drop comment "two source matches"
+		icmpv6 type nd-neighbor-solicit ip6 hoplimit != 255 drop comment "hop limit not 255"
+		iifname $iot_if udp dport 53 accept comment "DNS from the IoT VLAN"
+		ip saddr $admins jump admin
+	}
+
+	chain admin {
+		ip saddr 10.9.0.5 tcp dport 22 accept comment "within the jump"
+		ip saddr 10.8.0.1 tcp dport 23 accept comment "outside the jump"
+		meta mark 0x00000001 accept comment "raw under the jump"
+	}
+
+	chain forward {
+		type filter hook forward priority filter; policy drop;
+		ct state established,related accept
+		ip daddr 10.0.10.37 tcp dport 443 accept comment "not only forwarded"
+		iifname $lan_if oifname $wan_if accept
+	}
+
+	chain prerouting {
+		type nat hook prerouting priority dstnat; policy accept;
+		ip daddr 203.0.113.1 tcp dport { 80, 443 } dnat ip to 10.0.10.37 comment "web"
+	}
+}

internal/importer/testdata/audit.txt

@@ -0,0 +1,46 @@
+table inet audit { # handle 1
+	set blocklist { # handle 6
+		type ipv4_addr
+		timeout 1h
+	}
+
+	set lans { # handle 7
+		type ifname
+		elements = { "enp2s0",
+			     "enp3s0" }
+	}
+
+	chain early { # handle 1
+		type filter hook input priority filter - 10; policy accept;
+		ip saddr 192.0.2.66 drop comment "an early input chain" # handle 8
+	}
+
+	chain input { # handle 2
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept # handle 9
+		ip saddr @blocklist drop comment "dynamic" # handle 10
+		iifname @lans accept comment "an interface set" # handle 11
+		ip saddr 10.0.0.0/8 ip saddr != 10.0.10.0/24 tcp dport 22 drop comment "two source matches" # handle 12
+		icmpv6 type nd-neighbor-solicit ip6 hoplimit != 255 drop comment "hop limit not 255" # handle 13
+		iifname "enp2s0.20" udp dport 53 accept comment "DNS from the IoT VLAN" # handle 14
+		ip saddr 10.9.0.0/24 jump admin # handle 15
+	}
+
+	chain admin { # handle 3
+		ip saddr 10.9.0.5 tcp dport 22 accept comment "within the jump" # handle 16
+		ip saddr 10.8.0.1 tcp dport 23 accept comment "outside the jump" # handle 17
+		meta mark 0x00000001 accept comment "raw under the jump" # handle 18
+	}
+
+	chain forward { # handle 4
+		type filter hook forward priority filter; policy drop;
+		ct state established,related accept # handle 19
+		ip daddr 10.0.10.37 tcp dport 443 accept comment "not only forwarded" # handle 20
+		iifname "enp2s0" oifname "wan" accept # handle 21
+	}
+
+	chain prerouting { # handle 5
+		type nat hook prerouting priority dstnat; policy accept;
+		ip daddr 203.0.113.1 tcp dport { 80, 443 } dnat ip to 10.0.10.37 comment "web" # handle 23
+	}
+}

internal/importer/testdata/maps.json

@@ -0,0 +1 @@
+{"nftables": [{"metainfo": {"version": "1.1.3", "release_name": "Commodore Bullmoose #4", "json_schema_version": 1}}, {"table": {"family": "inet", "name": "main", "handle": 3}}, {"chain": {"family": "inet", "table": "main", "name": "web", "handle": 1}}, {"chain": {"family": "inet", "table": "main", "name": "prerouting", "handle": 2, "type": "nat", "hook": "prerouting", "prio": -100, "policy": "accept"}}, {"chain": {"family": "inet", "table": "main", "name": "input", "handle": 3, "type": "filter", "hook": "input", "prio": 0, "policy": "drop"}}, {"map": {"family": "inet", "name": "portforwards", "table": "main", "type": "inet_service", "handle": 4, "map": ["ipv4_addr", "inet_service"], "elem": [[80, {"concat": ["192.168.1.50", 8080]}], [443, {"concat": ["192.168.1.50", 443]}]]}}, {"map": {"family": "inet", "name": "hosts", "table": "main", "type": "inet_service", "handle": 5, "map": "ipv4_addr", "elem": [[2222, "192.168.1.60"]]}}, {"map": {"family": "inet", "name": "lan_services", "table": "main", "type": "inet_service", "handle": 6, "map": "verdict", "elem": [[22, {"accept": null}], [23, {"drop": null}], [80, {"jump": {"target": "web"}}]]}}, {"map": {"family": "inet", "name": "pairs", "table": "main", "type": ["ipv4_addr", "inet_service"], "handle": 7, "map": "verdict", "elem": [[{"concat": ["192.168.1.9", 22]}, {"accept": null}]]}}, {"rule": {"family": "inet", "table": "main", "chain": "web", "handle": 8, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": {"prefix": {"addr": "192.168.1.0", "len": 24}}}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "prerouting", "handle": 9, "comment": "Port forwards", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"dnat": {"family": "ip", "addr": {"map": {"key": {"payload": {"protocol": "tcp", "field": "dport"}}, "data": "@portforwards"}}}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "prerouting", "handle": 10, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"dnat": {"family": "ip", "addr": {"map": {"key": {"payload": {"protocol": "tcp", "field": "dport"}}, "data": "@hosts"}}}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "prerouting", "handle": 12, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"dnat": {"family": "ip", "addr": {"map": {"key": {"payload": {"protocol": "udp", "field": "dport"}}, "data": {"set": [[5000, {"concat": ["192.168.1.70", 5001]}], [6000, {"concat": ["192.168.1.71", 6000]}]]}}}}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 13, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": ["established", "related"]}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 14, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "lan"}}, {"vmap": {"key": {"payload": {"protocol": "tcp", "field": "dport"}}, "data": "@lan_services"}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 16, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"vmap": {"key": {"payload": {"protocol": "udp", "field": "dport"}}, "data": {"set": [[53, {"accept": null}], [123, {"drop": null}]]}}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 18, "expr": [{"vmap": {"key": {"payload": {"protocol": "ip", "field": "saddr"}}, "data": {"set": [[{"prefix": {"addr": "10.0.0.0", "len": 8}}, {"accept": null}], ["192.0.2.1", {"drop": null}]]}}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 19, "expr": [{"vmap": {"key": {"concat": [{"payload": {"protocol": "ip", "field": "saddr"}}, {"payload": {"protocol": "tcp", "field": "dport"}}]}, "data": "@pairs"}}]}}]}

internal/importer/testdata/maps.nft

@@ -0,0 +1,36 @@
+# Maps and verdict maps, as found in hand-written rulesets (importer_test.go).
+table inet main {
+	map portforwards {
+		type inet_service : ipv4_addr . inet_service
+		elements = { 443 : 192.168.1.50 . 443, 80 : 192.168.1.50 . 8080 }
+	}
+	map hosts {
+		type inet_service : ipv4_addr
+		elements = { 2222 : 192.168.1.60 }
+	}
+	map lan_services {
+		type inet_service : verdict
+		elements = { 22 : accept, 23 : drop, 80 : jump web }
+	}
+	map pairs {
+		type ipv4_addr . inet_service : verdict
+		elements = { 192.168.1.9 . 22 : accept }
+	}
+	chain web {
+		ip saddr 192.168.1.0/24 accept
+	}
+	chain prerouting {
+		type nat hook prerouting priority dstnat; policy accept;
+		iifname "wan" dnat ip to tcp dport map @portforwards comment "Port forwards"
+		iifname "wan" dnat ip to tcp dport map @hosts
+		iifname "wan" dnat ip to udp dport map { 5000 : 192.168.1.70 . 5001, 6000 : 192.168.1.71 . 6000 }
+	}
+	chain input {
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept
+		iifname "lan" tcp dport vmap @lan_services
+		iifname "wan" udp dport vmap { 53 : accept, 123 : drop }
+		ip saddr vmap { 10.0.0.0/8 : accept, 192.0.2.1 : drop }
+		ip saddr . tcp dport vmap @pairs
+	}
+}

internal/importer/testdata/maps.txt

@@ -0,0 +1,44 @@
+table inet main { # handle 3
+	map portforwards { # handle 4
+		type inet_service : ipv4_addr . inet_service
+		elements = { 80 : 192.168.1.50 . 8080,
+			     443 : 192.168.1.50 . 443 }
+	}
+
+	map hosts { # handle 5
+		type inet_service : ipv4_addr
+		elements = { 2222 : 192.168.1.60 }
+	}
+
+	map lan_services { # handle 6
+		type inet_service : verdict
+		elements = { 22 : accept,
+			     23 : drop,
+			     80 : jump web }
+	}
+
+	map pairs { # handle 7
+		type ipv4_addr . inet_service : verdict
+		elements = { 192.168.1.9 . 22 : accept }
+	}
+
+	chain web { # handle 1
+		ip saddr 192.168.1.0/24 accept # handle 8
+	}
+
+	chain prerouting { # handle 2
+		type nat hook prerouting priority dstnat; policy accept;
+		iifname "wan" dnat ip to tcp dport map @portforwards comment "Port forwards" # handle 9
+		iifname "wan" dnat ip to tcp dport map @hosts # handle 10
+		iifname "wan" dnat ip to udp dport map { 5000 : 192.168.1.70 . 5001, 6000 : 192.168.1.71 . 6000 } # handle 12
+	}
+
+	chain input { # handle 3
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept # handle 13
+		iifname "lan" tcp dport vmap @lan_services # handle 14
+		iifname "wan" udp dport vmap { 53 : accept, 123 : drop } # handle 16
+		ip saddr vmap { 10.0.0.0/8 : accept, 192.0.2.1 : drop } # handle 18
+		ip saddr . tcp dport vmap @pairs # handle 19
+	}
+}

internal/importer/testdata/meters.json

@@ -0,0 +1 @@
+{"nftables": [{"metainfo": {"version": "1.1.3", "release_name": "Commodore Bullmoose #4", "json_schema_version": 1}}, {"table": {"family": "inet", "name": "main", "handle": 1}}, {"chain": {"family": "inet", "table": "main", "name": "input", "handle": 1, "type": "filter", "hook": "input", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "main", "name": "forward", "handle": 2, "type": "filter", "hook": "forward", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "main", "name": "output", "handle": 3, "type": "filter", "hook": "output", "prio": 0, "policy": "accept"}}, {"set": {"family": "inet", "name": "ssh_meter", "table": "main", "type": "ipv4_addr", "handle": 4, "size": 65535, "flags": ["timeout", "dynamic"], "timeout": 60}}, {"set": {"family": "inet", "name": "ssh6_meter", "table": "main", "type": "ipv6_addr", "handle": 5, "size": 65535, "flags": ["timeout", "dynamic"], "timeout": 60}}, {"set": {"family": "inet", "name": "my_connlimit", "table": "main", "type": "ipv4_addr", "handle": 6, "size": 65535, "flags": ["dynamic"]}}, {"set": {"family": "inet", "name": "guest_quota", "table": "main", "type": "ipv4_addr", "handle": 7, "size": 65535, "flags": ["dynamic"]}}, {"set": {"family": "inet", "name": "trusted", "table": "main", "type": "ipv4_addr", "handle": 8, "flags": ["interval"], "elem": [{"prefix": {"addr": "10.0.10.0", "len": 24}}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 9, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": ["established", "related"]}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 10, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 22}}, {"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": "new"}}, {"set": {"op": "update", "elem": {"payload": {"protocol": "ip", "field": "saddr"}}, "set": "@ssh_meter", "stmt": [{"limit": {"rate": 3, "burst": 5, "per": "minute", "inv": true}}]}}, {"counter": {"packets": 0, "bytes": 0}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 11, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 22}}, {"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": "new"}}, {"set": {"op": "update", "elem": {"&": [{"payload": {"protocol": "ip6", "field": "saddr"}}, "ffff:ffff:ffff:ffff::"]}, "set": "@ssh6_meter", "stmt": [{"limit": {"rate": 3, "burst": 5, "per": "minute", "inv": true}}]}}, {"counter": {"packets": 0, "bytes": 0}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 12, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": "@trusted"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": 22}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "forward", "handle": 13, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": "new"}}, {"set": {"op": "add", "elem": {"payload": {"protocol": "ip", "field": "saddr"}}, "set": "@my_connlimit", "stmt": [{"ct count": {"val": 20, "inv": true}}]}}, {"counter": {"packets": 0, "bytes": 0}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "forward", "handle": 14, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "lan"}}, {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": "wan"}}, {"limit": {"rate": 10, "burst": 512, "per": "second", "inv": true, "rate_unit": "mbytes", "burst_unit": "kbytes"}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "forward", "handle": 15, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "guest0"}}, {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": "wan"}}, {"set": {"op": "add", "elem": {"&": [{"payload": {"protocol": "ip", "field": "saddr"}}, "255.255.255.0"]}, "set": "@guest_quota", "stmt": [{"quota": {"val": 1000, "val_unit": "mbytes", "inv": true}}]}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "forward", "handle": 16, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "lan"}}, {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": "wan"}}, {"quota": {"val": 100000, "val_unit": "mbytes", "inv": true}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "forward", "handle": 17, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "lan"}}, {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": "wan"}}, {"accept": null}]}}]}

internal/importer/testdata/meters.nft

@@ -0,0 +1,46 @@
+# Meters, byte rates and quotas, as hand-written rulesets use them (importer_test.go).
+table inet main {
+	set ssh_meter {
+		type ipv4_addr
+		flags dynamic,timeout
+		timeout 1m
+	}
+	set ssh6_meter {
+		type ipv6_addr
+		flags dynamic,timeout
+		timeout 1m
+	}
+	set my_connlimit {
+		type ipv4_addr
+		size 65535
+		flags dynamic
+	}
+	set guest_quota {
+		type ipv4_addr
+		size 65535
+		flags dynamic
+	}
+	set trusted {
+		type ipv4_addr
+		flags interval
+		elements = { 10.0.10.0/24 }
+	}
+	chain input {
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept
+		tcp dport 22 ct state new update @ssh_meter { ip saddr limit rate over 3/minute burst 5 packets } counter drop
+		tcp dport 22 ct state new update @ssh6_meter { ip6 saddr and ffff:ffff:ffff:ffff:: limit rate over 3/minute } counter drop
+		ip saddr @trusted tcp dport 22 accept
+	}
+	chain forward {
+		type filter hook forward priority filter; policy drop;
+		ct state new add @my_connlimit { ip saddr ct count over 20 } counter drop
+		iifname "lan" oifname "wan" limit rate over 10 mbytes/second burst 512 kbytes drop
+		iifname "guest0" oifname "wan" add @guest_quota { ip saddr and 255.255.255.0 quota over 1000 mbytes } drop
+		iifname "lan" oifname "wan" quota over 100000 mbytes drop
+		iifname "lan" oifname "wan" accept
+	}
+	chain output {
+		type filter hook output priority filter; policy accept;
+	}
+}

internal/importer/testdata/meters.txt

@@ -0,0 +1,54 @@
+table inet main { # handle 1
+	set ssh_meter { # handle 4
+		type ipv4_addr
+		size 65535
+		flags dynamic,timeout
+		timeout 1m
+	}
+
+	set ssh6_meter { # handle 5
+		type ipv6_addr
+		size 65535
+		flags dynamic,timeout
+		timeout 1m
+	}
+
+	set my_connlimit { # handle 6
+		type ipv4_addr
+		size 65535
+		flags dynamic
+	}
+
+	set guest_quota { # handle 7
+		type ipv4_addr
+		size 65535
+		flags dynamic
+	}
+
+	set trusted { # handle 8
+		type ipv4_addr
+		flags interval
+		elements = { 10.0.10.0/24 }
+	}
+
+	chain input { # handle 1
+		type filter hook input priority filter; policy drop;
+		ct state established,related accept # handle 9
+		tcp dport 22 ct state new update @ssh_meter { ip saddr limit rate over 3/minute burst 5 packets } counter packets 0 bytes 0 drop # handle 10
+		tcp dport 22 ct state new update @ssh6_meter { ip6 saddr & ffff:ffff:ffff:ffff:: limit rate over 3/minute burst 5 packets } counter packets 0 bytes 0 drop # handle 11
+		ip saddr @trusted tcp dport 22 accept # handle 12
+	}
+
+	chain forward { # handle 2
+		type filter hook forward priority filter; policy drop;
+		ct state new add @my_connlimit { ip saddr ct count over 20 } counter packets 0 bytes 0 drop # handle 13
+		iifname "lan" oifname "wan" limit rate over 10 mbytes/second burst 512 kbytes drop # handle 14
+		iifname "guest0" oifname "wan" add @guest_quota { ip saddr & 255.255.255.0 quota over 1000 mbytes } drop # handle 15
+		iifname "lan" oifname "wan" quota over 100000 mbytes drop # handle 16
+		iifname "lan" oifname "wan" accept # handle 17
+	}
+
+	chain output { # handle 3
+		type filter hook output priority filter; policy accept;
+	}
+}

internal/importer/testdata/router.json

@@ -0,0 +1 @@
+{"nftables": [{"metainfo": {"version": "1.1.3", "release_name": "Commodore Bullmoose #4", "json_schema_version": 1}}, {"table": {"family": "inet", "name": "main", "handle": 1}}, {"chain": {"family": "inet", "table": "main", "name": "antispoof", "handle": 1, "type": "filter", "hook": "prerouting", "prio": -300, "policy": "accept"}}, {"chain": {"family": "inet", "table": "main", "name": "input", "handle": 2, "type": "filter", "hook": "input", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "main", "name": "wg0_input", "handle": 3}}, {"chain": {"family": "inet", "table": "main", "name": "prerouting", "handle": 4, "type": "nat", "hook": "prerouting", "prio": -100, "policy": "accept"}}, {"chain": {"family": "inet", "table": "main", "name": "postrouting", "handle": 5, "type": "nat", "hook": "postrouting", "prio": 100, "policy": "accept"}}, {"chain": {"family": "inet", "table": "main", "name": "forward", "handle": 6, "type": "filter", "hook": "forward", "prio": 0, "policy": "drop"}}, {"chain": {"family": "inet", "table": "main", "name": "output", "handle": 7, "type": "filter", "hook": "output", "prio": 0, "policy": "accept"}}, {"rule": {"family": "inet", "table": "main", "chain": "antispoof", "handle": 9, "comment": "Private/bogus source addresses can't legitimately arrive from the internet", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": {"set": [{"prefix": {"addr": "0.0.0.0", "len": 8}}, {"prefix": {"addr": "10.0.0.0", "len": 8}}, {"prefix": {"addr": "127.0.0.0", "len": 8}}, {"prefix": {"addr": "169.254.0.0", "len": 16}}, {"prefix": {"addr": "172.16.0.0", "len": 12}}, {"prefix": {"addr": "192.168.0.0", "len": 16}}]}}}, {"counter": {"packets": 0, "bytes": 0}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "antispoof", "handle": 10, "comment": "Our own delegated prefix can't legitimately arrive from the internet", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "ip6", "field": "saddr"}}, "right": {"prefix": {"addr": "2a02:8013:7f3d::", "len": 48}}}}, {"counter": {"packets": 0, "bytes": 0}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 11, "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "lo"}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 12, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": ["established", "related"]}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 13, "expr": [{"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": "invalid"}}, {"drop": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 15, "expr": [{"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "icmp", "field": "type"}}, "right": {"set": ["echo-reply", "echo-request"]}}}, {"counter": {"packets": 0, "bytes": 0}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 17, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "icmpv6", "field": "type"}}, "right": {"set": ["destination-unreachable", "packet-too-big", "time-exceeded", "parameter-problem", "echo-request", "echo-reply"]}}}, {"counter": {"packets": 0, "bytes": 0}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 19, "comment": "IPv6 neighbour discovery", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "icmpv6", "field": "type"}}, "right": {"set": ["nd-router-solicit", "nd-router-advert", "nd-neighbor-solicit", "nd-neighbor-advert"]}}}, {"match": {"op": "==", "left": {"payload": {"protocol": "ip6", "field": "hoplimit"}}, "right": 255}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 20, "comment": "DHCPv6 replies from Zen", "expr": [{"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": "wan"}}, {"match": {"op": "==", "left": {"payload": {"protocol": "udp", "field": "dport"}}, "right": 546}}, {"match": {"op": "==", "left": {"payload": {"protocol": "ip6", "field": "saddr"}}, "right": {"prefix": {"addr": "fe80::", "len": 10}}}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 21, "comment": "Accept new incoming wireguard connections", "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "udp", "field": "dport"}}, "right": 51820}}, {"match": {"op": "in", "left": {"ct": {"key": "state"}}, "right": "new"}}, {"accept": null}]}}, {"rule": {"family": "inet", "table": "main", "chain": "input", "handle": 22, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "Line truncated

internal/importer/testdata/router.nft

@@ -0,0 +1,142 @@
+#! /sbin/nft -f
+
+flush ruleset
+
+define wan_int = "wan"
+define wan_ip = 217.155.8.194
+define allowed_icmp_types = { echo-request, echo-reply }
+define allowed_icmpv6_types = { destination-unreachable, packet-too-big, time-exceeded, parameter-problem, echo-request, echo-reply }
+define trusted_ips = { 10.0.10.0/24, 10.20.30.0/24, 10.55.55.0/24, 10.1.1.0/24, 10.65.65.0/24, 10.60.60.0/24, 10.50.50.0/24}
+define masq_to_internet = { 10.0.10.0/24, 10.20.30.0/24, 10.55.55.0/24, 10.1.1.0/24, 10.65.65.0/24, 10.60.60.0/24 }
+define bogons = { 0.0.0.0/8, 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16 }
+define internal_net = 10.0.10.0/24
+define lan6_net = 2a02:8013:7f3d:10::/64
+define my_prefix6 = 2a02:8013:7f3d::/48
+define wg0_int = wg0
+define wg0_ip = 10.50.50.1
+define wg0_net = 10.50.50.0/24
+define ekko_wg0 = 10.50.50.2
+define epcc_wg0 = 10.50.50.3
+define oneplus9_wg0 = 10.50.50.4
+define dhcp_int = enp2s0
+define heimer_incus = 10.20.30.0/24
+define garen_incus = 10.55.55.0/24
+define jellyfin_ip = 10.0.10.57
+define jellyfin_port = 8096
+define haproxy_ip = 10.0.10.37
+define checkmk_server = 10.20.30.58
+define oracle_wg0 = 10.50.50.6/32
+define slskd_ip = 10.0.10.35
+define slskd_port = 50300
+define terraria_port = { 7777 }
+define ula_prefix = fdab:abab:abab::/48
+define lan6_ula = fdab:abab:abab:10::/64
+define dot_doh_ports = { 8443, 853 }
+define dot_doh_ip4 = { 10.0.10.5, 217.155.8.194 }
+define dot_doh_ip6 = { fdab:abab:abab:10::5, 2a02:8013:7f3d:10::5 }
+
+define jf_ipv6 = 2a02:8013:7f3d:10::57
+
+table inet main {
+        chain antispoof {
+                type filter hook prerouting priority raw; policy accept;
+                iifname $wan_int ip saddr $bogons counter drop comment "Private/bogus source addresses can't legitimately arrive from the internet"
+                iifname $wan_int ip6 saddr $my_prefix6 counter drop comment "Our own delegated prefix can't legitimately arrive from the internet"
+        }
+
+        chain input {
+                type filter hook input priority filter; policy drop;
+
+                iifname lo accept
+
+                ct state established,related accept
+                ct state invalid drop
+
+                iifname != $wan_int icmp type $allowed_icmp_types counter accept
+                icmpv6 type $allowed_icmpv6_types counter accept
+                icmpv6 type { nd-router-solicit, nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 255 accept comment "IPv6 neighbour discovery"
+                iifname $wan_int udp dport 546 ip6 saddr fe80::/10 accept comment "DHCPv6 replies from Zen"
+
+                udp dport 51820 ct state new accept comment "Accept new incoming wireguard connections"
+
+                ip saddr $wg0_net jump wg0_input
+
+                udp dport 53 accept comment "Accept DNS Traffic UDP"
+                tcp dport 53 accept comment "Accept DNS Traffic TCP"
+                tcp dport $dot_doh_ports ip daddr $dot_doh_ip4 accept comment "Accept DoT and DoH Traffic"
+                tcp dport $dot_doh_ports ip6 daddr $dot_doh_ip6 accept comment "Accept DoT and DoH Traffic"
+                #ip saddr $trusted_ips udp dport 53 accept comment "Accept DNS Traffic UDP"
+                #ip saddr $trusted_ips tcp dport 53 accept comment "Accept DNS Traffic TCP"
+                ip saddr $trusted_ips tcp dport 8053 accept comment "Accept Jarl traffic TCP"
+                ip saddr $trusted_ips tcp dport 8067 accept comment "Accept Steward traffic TCP"
+                ip saddr $trusted_ips tcp dport 22 accept
+
+                ip6 saddr $lan6_net udp dport 53 accept comment "Accept DNS Traffic UDP (IPv6)"
+                ip6 saddr $lan6_net tcp dport 53 accept comment "Accept DNS Traffic TCP (IPv6)"
+                ip6 saddr $lan6_net tcp dport 22 accept
+
+                iifname $dhcp_int udp dport 67 accept comment "Accept DHCP"
+                iifname $dhcp_int udp dport 547 ip6 saddr fe80::/10 accept comment "Accept DHCPv6"
+                ip saddr $checkmk_server tcp dport 6556 accept comment "From Checkmk To Agent"
+        }
+
+        chain wg0_input {
+                udp dport 53 accept
+                tcp dport 53 accept
+                ip saddr { $epcc_wg0, $oracle_wg0 } tcp dport 22 accept
+                ip saddr $wg0_net ip daddr $wg0_ip drop comment "Block WireGuard clients from accessing the server itself"
+                return
+        }
+
+        chain prerouting {
+                type nat hook prerouting priority dstnat; policy accept;
+                # Matching on the public IP (not the WAN interface) covers both internet traffic and hairpin traffic from inside
+                ip daddr $wan_ip tcp dport $jellyfin_port dnat ip to $jellyfin_ip:443 comment "Port Forward Jellyfin"
+                ip daddr $wan_ip tcp dport 443 dnat ip to $haproxy_ip:443 comment "Port Forward Haproxy"
+                ip daddr $wan_ip tcp dport $slskd_port dnat ip to $slskd_ip:$slskd_port comment "Port Forward soulseek"
+                ip daddr $wan_ip tcp dport $terraria_port dnat ip to 10.0.10.76 comment "Port Forward terraria"
+                ip daddr $wan_ip tcp dport 25 dnat ip to 10.0.10.25 comment "Port Forward SMTP"
+        }
+
+        chain postrouting {
+                type nat hook postrouting priority srcnat; policy accept;
+                oifname $wan_int ip saddr $masq_to_internet masquerade comment "external masquerade"
+                oifname != $wg0_int ip saddr $wg0_net masquerade
+                # Same-subnet clients would otherwise get replies direct from the server, bypassing the router's NAT
+                ip saddr $internal_net ip daddr { $haproxy_ip, $jellyfin_ip, $slskd_ip, 10.0.10.76, 10.0.10.25 } ct status dnat masquerade comment "Hairpin NAT same-subnet"
+        }
+
+        chain forward {
+                type filter hook forward priority filter; policy drop;
+
+                # PPPoE MTU is smaller than ethernet; must sit before the established rule so SYN-ACKs get clamped too
+                oifname $wan_int tcp flags syn tcp option maxseg size set rt mtu comment "PPPoE MSS clamping"
+
+                ct state established,related accept
+                iifname != $wan_int ct state invalid ip daddr $trusted_ips accept comment "Needed due to asymmetric routing (internal only)"
+                ct state invalid drop
+
+                iifname $dhcp_int oifname $wan_int ip6 saddr $lan6_net accept comment "LAN IPv6 out"
+
+                # From the internet, only allow traffic that was port forwarded
+                iifname $wan_int ip daddr $jellyfin_ip tcp dport 443 ct status dnat accept comment "Forward jellyfin traffic"
+                iifname $wan_int ip daddr $haproxy_ip tcp dport 443 ct status dnat accept comment "Forward Haproxy traffic"
+                iifname $wan_int ip daddr $slskd_ip tcp dport $slskd_port ct status dnat accept comment "Forward slskd traffic"
+                iifname $wan_int ip daddr 10.0.10.76 tcp dport $terraria_port ct status dnat accept comment "Forward terraria traffic"
+                iifname $wan_int ip daddr 10.0.10.25 tcp dport 25 ct status dnat accept comment "Forward SMTP"
+                ip6 daddr $jf_ipv6 tcp dport 443 accept
+
+                iifname $wan_int counter drop comment "Nothing else gets in from the internet"
+
+                ip daddr $wg0_net drop comment "Drop client-to-client traffic for wg0 network"
+                icmp type $allowed_icmp_types accept
+
+                ip saddr $wg0_net accept comment "Forward from wg0 network"
+                ip saddr $trusted_ips accept comment "Forward from trusted IPs"
+
+        }
+
+        chain output {
+                type filter hook output priority filter; policy accept;
+        }
+}

internal/importer/testdata/router.txt

@@ -0,0 +1,81 @@
+table inet main { # handle 1
+	chain antispoof { # handle 1
+		type filter hook prerouting priority raw; policy accept;
+		iifname "wan" ip saddr { 0.0.0.0/8, 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16 } counter packets 0 bytes 0 drop comment "Private/bogus source addresses can't legitimately arrive from the internet" # handle 9
+		iifname "wan" ip6 saddr 2a02:8013:7f3d::/48 counter packets 0 bytes 0 drop comment "Our own delegated prefix can't legitimately arrive from the internet" # handle 10
+	}
+
+	chain input { # handle 2
+		type filter hook input priority filter; policy drop;
+		iifname "lo" accept # handle 11
+		ct state established,related accept # handle 12
+		ct state invalid drop # handle 13
+		iifname != "wan" icmp type { echo-reply, echo-request } counter packets 0 bytes 0 accept # handle 15
+		icmpv6 type { destination-unreachable, packet-too-big, time-exceeded, parameter-problem, echo-request, echo-reply } counter packets 0 bytes 0 accept # handle 17
+		icmpv6 type { nd-router-solicit, nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 255 accept comment "IPv6 neighbour discovery" # handle 19
+		iifname "wan" udp dport 546 ip6 saddr fe80::/10 accept comment "DHCPv6 replies from Zen" # handle 20
+		udp dport 51820 ct state new accept comment "Accept new incoming wireguard connections" # handle 21
+		ip saddr 10.50.50.0/24 jump wg0_input # handle 22
+		udp dport 53 accept comment "Accept DNS Traffic UDP" # handle 23
+		tcp dport 53 accept comment "Accept DNS Traffic TCP" # handle 24
+		tcp dport { 853, 8443 } ip daddr { 10.0.10.5, 217.155.8.194 } accept comment "Accept DoT and DoH Traffic" # handle 27
+		tcp dport { 853, 8443 } ip6 daddr { 2a02:8013:7f3d:10::5, fdab:abab:abab:10::5 } accept comment "Accept DoT and DoH Traffic" # handle 30
+		ip saddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.50.50.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } tcp dport 8053 accept comment "Accept Jarl traffic TCP" # handle 32
+		ip saddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.50.50.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } tcp dport 8067 accept comment "Accept Steward traffic TCP" # handle 34
+		ip saddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.50.50.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } tcp dport 22 accept # handle 36
+		ip6 saddr 2a02:8013:7f3d:10::/64 udp dport 53 accept comment "Accept DNS Traffic UDP (IPv6)" # handle 37
+		ip6 saddr 2a02:8013:7f3d:10::/64 tcp dport 53 accept comment "Accept DNS Traffic TCP (IPv6)" # handle 38
+		ip6 saddr 2a02:8013:7f3d:10::/64 tcp dport 22 accept # handle 39
+		iifname "enp2s0" udp dport 67 accept comment "Accept DHCP" # handle 40
+		iifname "enp2s0" udp dport 547 ip6 saddr fe80::/10 accept comment "Accept DHCPv6" # handle 41
+		ip saddr 10.20.30.58 tcp dport 6556 accept comment "From Checkmk To Agent" # handle 42
+	}
+
+	chain wg0_input { # handle 3
+		udp dport 53 accept # handle 43
+		tcp dport 53 accept # handle 44
+		ip saddr { 10.50.50.3, 10.50.50.6 } tcp dport 22 accept # handle 46
+		ip saddr 10.50.50.0/24 ip daddr 10.50.50.1 drop comment "Block WireGuard clients from accessing the server itself" # handle 47
+		return # handle 48
+	}
+
+	chain prerouting { # handle 4
+		type nat hook prerouting priority dstnat; policy accept;
+		ip daddr 217.155.8.194 tcp dport 8096 dnat ip to 10.0.10.57:443 comment "Port Forward Jellyfin" # handle 49
+		ip daddr 217.155.8.194 tcp dport 443 dnat ip to 10.0.10.37:443 comment "Port Forward Haproxy" # handle 50
+		ip daddr 217.155.8.194 tcp dport 50300 dnat ip to 10.0.10.35:50300 comment "Port Forward soulseek" # handle 51
+		ip daddr 217.155.8.194 tcp dport 7777 dnat ip to 10.0.10.76 comment "Port Forward terraria" # handle 52
+		ip daddr 217.155.8.194 tcp dport 25 dnat ip to 10.0.10.25 comment "Port Forward SMTP" # handle 53
+	}
+
+	chain postrouting { # handle 5
+		type nat hook postrouting priority srcnat; policy accept;
+		oifname "wan" ip saddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } masquerade comment "external masquerade" # handle 55
+		oifname != "wg0" ip saddr 10.50.50.0/24 masquerade # handle 56
+		ip saddr 10.0.10.0/24 ip daddr { 10.0.10.25, 10.0.10.35, 10.0.10.37, 10.0.10.57, 10.0.10.76 } ct status dnat masquerade comment "Hairpin NAT same-subnet" # handle 58
+	}
+
+	chain forward { # handle 6
+		type filter hook forward priority filter; policy drop;
+		oifname "wan" tcp flags syn tcp option maxseg size set rt mtu comment "PPPoE MSS clamping" # handle 59
+		ct state established,related accept # handle 60
+		iifname != "wan" ct state invalid ip daddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.50.50.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } accept comment "Needed due to asymmetric routing (internal only)" # handle 62
+		ct state invalid drop # handle 63
+		iifname "enp2s0" oifname "wan" ip6 saddr 2a02:8013:7f3d:10::/64 accept comment "LAN IPv6 out" # handle 64
+		iifname "wan" ip daddr 10.0.10.57 tcp dport 443 ct status dnat accept comment "Forward jellyfin traffic" # handle 65
+		iifname "wan" ip daddr 10.0.10.37 tcp dport 443 ct status dnat accept comment "Forward Haproxy traffic" # handle 66
+		iifname "wan" ip daddr 10.0.10.35 tcp dport 50300 ct status dnat accept comment "Forward slskd traffic" # handle 67
+		iifname "wan" ip daddr 10.0.10.76 tcp dport 7777 ct status dnat accept comment "Forward terraria traffic" # handle 68
+		iifname "wan" ip daddr 10.0.10.25 tcp dport 25 ct status dnat accept comment "Forward SMTP" # handle 69
+		ip6 daddr 2a02:8013:7f3d:10::57 tcp dport 443 accept # handle 70
+		iifname "wan" counter packets 0 bytes 0 drop comment "Nothing else gets in from the internet" # handle 71
+		ip daddr 10.50.50.0/24 drop comment "Drop client-to-client traffic for wg0 network" # handle 72
+		icmp type { echo-reply, echo-request } accept # handle 74
+		ip saddr 10.50.50.0/24 accept comment "Forward from wg0 network" # handle 75
+		ip saddr { 10.0.10.0/24, 10.1.1.0/24, 10.20.30.0/24, 10.50.50.0/24, 10.55.55.0/24, 10.60.60.0/24, 10.65.65.0/24 } accept comment "Forward from trusted IPs" # handle 77
+	}
+
+	chain output { # handle 7
+		type filter hook output priority filter; policy accept;
+	}
+}

internal/jarl/jarl.go

@@ -0,0 +1,278 @@
+// Package jarl talks to Jarl, the BIND manager, through its REST API: who
+// the token belongs to, which zones it can see, and DNS-01 challenge
+// records. It writes challenge records the way 'jarl acme' does: into the
+// longest matching primary zone, in every view that has it (once per set
+// of linked zones), so the CA and internal resolvers see the same thing.
+package jarl
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"housecarl/internal/version"
+)
+
+// ChallengeTTL is the TTL of challenge records (as in Jarl).
+const ChallengeTTL = 60
+
+// Comment marks the challenge records Housecarl writes.
+const Comment = "ACME DNS-01 challenge (Housecarl)"
+
+// Client is a Jarl server and a token.
+type Client struct {
+	base  string
+	token string
+	http  *http.Client
+}
+
+// New makes a client. caCert is a PEM file of a CA to trust ("" for the
+// system's).
+func New(server, token, caCert string, insecure bool) (*Client, error) {
+	tc := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: insecure} //nolint:gosec // the user's choice
+	if caCert != "" {
+		pem, err := os.ReadFile(caCert)
+		if err != nil {
+			return nil, fmt.Errorf("jarl.ca_cert: %w", err)
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil || pool == nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("jarl.ca_cert: no certificates in %s", caCert)
+		}
+		tc.RootCAs = pool
+	}
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = tc
+	return &Client{base: strings.TrimSuffix(server, "/") + "/api/v1", token: token, http: &http.Client{Transport: tr, Timeout: 2 * time.Minute}}, nil
+}
+
+// Error is an error answer from Jarl.
+type Error struct {
+	Status  int
+	Code    string
+	Message string
+}
+
+func (e *Error) Error() string { return fmt.Sprintf("Jarl answered %d: %s", e.Status, e.Message) }
+
+func (c *Client) do(ctx context.Context, method, path string, body, out any) error {
+	var rd io.Reader
+	if body != nil {
+		b, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		rd = bytes.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, c.base+path, rd)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "housecarl/"+version.Short())
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("can't reach Jarl at %s: %w", strings.TrimSuffix(c.base, "/api/v1"), err)
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		var eb struct {
+			Error struct {
+				Code    string `json:"code"`
+				Message string `json:"message"`
+			} `json:"error"`
+		}
+		e := &Error{Status: resp.StatusCode, Message: strings.TrimSpace(string(b))}
+		if json.Unmarshal(b, &eb) == nil && eb.Error.Message != "" {
+			e.Code, e.Message = eb.Error.Code, eb.Error.Message
+		}
+		return e
+	}
+	if out != nil {
+		return json.Unmarshal(b, out)
+	}
+	return nil
+}
+
+// Whoami is who the token belongs to.
+type Whoami struct {
+	User struct {
+		Username string `json:"username"`
+	} `json:"user"`
+	Admin       bool     `json:"admin"`
+	Method      string   `json:"method"`
+	Permissions []string `json:"permissions"`
+	Build       struct {
+		Label string `json:"label"`
+	} `json:"build"`
+}
+
+// Whoami asks Jarl who the token is.
+func (c *Client) Whoami(ctx context.Context) (*Whoami, error) {
+	var w Whoami
+	return &w, c.do(ctx, http.MethodGet, "/auth/whoami", nil, &w)
+}
+
+// Zone is a primary zone the token can see.
+type Zone struct {
+	Name      string `json:"name"` // "adbraun.lol."
+	View      string `json:"view,omitempty"`
+	LinkGroup int64  `json:"link_group,omitempty"`
+}
+
+// Zones lists the primary zones the token can see.
+func (c *Client) Zones(ctx context.Context) ([]Zone, error) {
+	var zs []Zone
+	return zs, c.do(ctx, http.MethodGet, "/zones?type=primary", nil, &zs)
+}
+
+func fqdn(n string) string {
+	n = strings.ToLower(strings.TrimSpace(n))
+	if !strings.HasSuffix(n, ".") {
+		n += "."
+	}
+	return n
+}
+
+// ChallengeName is the record name for a domain's DNS-01 challenge.
+func ChallengeName(domain string) string {
+	d := strings.TrimPrefix(strings.ToLower(strings.TrimSuffix(domain, ".")), "*.")
+	if !strings.HasPrefix(d, "_acme-challenge.") {
+		d = "_acme-challenge." + d
+	}
+	return d + "."
+}
+
+// ZonesFor picks the zones a record at name goes into: the longest
+// matching zone, in every view that has it (or only view), once per set of
+// linked zones.
+func ZonesFor(name string, zones []Zone, view string) []Zone {
+	name = fqdn(name)
+	best := ""
+	var out []Zone
+	for _, z := range zones {
+		if view != "" && !strings.EqualFold(z.View, view) {
+			continue
+		}
+		zn := fqdn(z.Name)
+		if name != zn && !strings.HasSuffix(name, "."+zn) && zn != "." {
+			continue
+		}
+		switch {
+		case len(zn) > len(best):
+			best, out = zn, []Zone{z}
+		case zn == best:
+			out = append(out, z)
+		}
+	}
+	var dedup []Zone
+	links := map[int64]bool{}
+	for _, z := range out {
+		if z.LinkGroup != 0 {
+			if links[z.LinkGroup] {
+				continue
+			}
+			links[z.LinkGroup] = true
+		}
+		dedup = append(dedup, z)
+	}
+	return dedup
+}
+
+type op struct {
+	Op   string `json:"op"`
+	Data any    `json:"data"`
+}
+
+type batch struct {
+	Description string `json:"description"`
+	Ops         []op   `json:"ops"`
+	DryRun      bool   `json:"dry_run,omitempty"`
+}
+
+// SetTXT publishes (set) or removes a TXT value at name in zones, as one
+// change in Jarl. Setting first removes the same value, so a retry
+// succeeds; removing a value that isn't there succeeds too. With dryRun
+// Jarl only checks the change, including the token's permission.
+func (c *Client) SetTXT(ctx context.Context, name, value string, zones []Zone, set, dryRun bool, why string) error {
+	if len(zones) == 0 {
+		return errors.New("no zone to write into")
+	}
+	var ops []op
+	for _, z := range zones {
+		base := map[string]any{"zone": z.Name, "name": fqdn(name), "type": "TXT", "data": value}
+		if z.View != "" {
+			base["view"] = z.View
+		}
+		del := map[string]any{"missing_ok": true}
+		for k, v := range base {
+			del[k] = v
+		}
+		ops = append(ops, op{Op: "record.delete", Data: del})
+		if set {
+			add := map[string]any{"ttl": ChallengeTTL, "comment": Comment}
+			for k, v := range base {
+				add[k] = v
+			}
+			ops = append(ops, op{Op: "record.create", Data: add})
+		}
+	}
+	verb := "remove"
+	if set {
+		verb = "set"
+	}
+	desc := "ACME: " + verb + " challenge for " + strings.TrimSuffix(name, ".")
+	if why != "" {
+		desc += " (" + why + ")"
+	}
+	return c.do(ctx, http.MethodPost, "/apply", batch{Description: desc, Ops: ops, DryRun: dryRun}, nil)
+}
+
+// ZoneInfo is a zone as Jarl lists it, of any type.
+type ZoneInfo struct {
+	ID       int64  `json:"id"`
+	Name     string `json:"name"` // "example.com."
+	View     string `json:"view,omitempty"`
+	Type     string `json:"type"`
+	InView   string `json:"in_view,omitempty"` // type in-view: the view whose zone it serves
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+// AllZones lists every zone the token can see, of any type.
+func (c *Client) AllZones(ctx context.Context) ([]ZoneInfo, error) {
+	var zs []ZoneInfo
+	return zs, c.do(ctx, http.MethodGet, "/zones", nil, &zs)
+}
+
+// Record is a record as Jarl lists it.
+type Record struct {
+	Name string `json:"name"` // "nas.example.com."
+	Type string `json:"type"`
+	Data string `json:"data"`
+}
+
+// Records lists a zone's enabled records (zone: its ID).
+func (c *Client) Records(ctx context.Context, zone int64) ([]Record, error) {
+	var rs []Record
+	return rs, c.do(ctx, http.MethodGet, fmt.Sprintf("/zones/%d/records?disabled=0", zone), nil, &rs)
+}

internal/jarlsets/jarlsets.go

@@ -0,0 +1,767 @@
+// Package jarlsets fills the sets whose addresses come from Jarl. It reads
+// the A and AAAA records of the sets' names through Jarl's API, following
+// CNAMEs within the zones Jarl holds, every minute or so, and keeps what it
+// last read (in the database too) for when Jarl can't be reached. The
+// firewall gets, for each set, the addresses that lie within what the set
+// allows (zones' connected networks, or networks it names); the others are
+// left out and reported. When what the sets hold changes, the ruleset is
+// applied again (no history entry: the configuration didn't change).
+package jarlsets
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/netip"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"housecarl/internal/api"
+	"housecarl/internal/jarl"
+	"housecarl/internal/model"
+	"housecarl/internal/validate"
+)
+
+// MetaKey is the database meta key holding what was last read.
+const MetaKey = "jarl_sets"
+
+// maxChain is how many CNAMEs are followed for one name.
+const maxChain = 8
+
+// Source is Jarl (internal/jarl.Client).
+type Source interface {
+	AllZones(ctx context.Context) ([]jarl.ZoneInfo, error)
+	Records(ctx context.Context, zone int64) ([]jarl.Record, error)
+}
+
+// Lookup is what Jarl gave for one name in one view.
+type Lookup struct {
+	Name      string   `json:"name"`
+	View      string   `json:"view,omitempty"`
+	Found     string   `json:"found,omitempty"`
+	Chain     []string `json:"chain,omitempty"`
+	Addresses []string `json:"addresses,omitempty"`
+	// Problem is why Jarl's records give no addresses (Jarl answered).
+	Problem string `json:"problem,omitempty"`
+	// Checked is when Jarl last answered; Error why the last try failed.
+	Checked time.Time `json:"checked"`
+	Error   string    `json:"error,omitempty"`
+}
+
+// saved is what the database keeps.
+type saved struct {
+	Lookups map[string]*Lookup `json:"lookups"`
+	// Nets are the zones' connected networks last seen, for when a zone's
+	// interfaces have no address (at boot, a link down).
+	Nets map[string][]netip.Prefix `json:"nets,omitempty"`
+}
+
+// Resolver keeps the Jarl sets' addresses.
+type Resolver struct {
+	// Source is Jarl (nil: not configured; the sets keep what was read).
+	Source Source
+	// Interval is how often Jarl is read (default a minute).
+	Interval time.Duration
+	// State is the configuration in force.
+	State func() *model.State
+	// Connected are the networks connected to the firewall, with their
+	// zones.
+	Connected func(st *model.State) map[netip.Prefix]string
+	// Apply puts the sets' new addresses in force.
+	Apply func(ctx context.Context) error
+	// Audit records a change of a set's addresses.
+	Audit func(ctx context.Context, set, detail string)
+	// Load and Save read and write the database's copy.
+	Load func(ctx context.Context) (string, error)
+	Save func(ctx context.Context, value string) error
+	Log  *slog.Logger
+
+	run  sync.Mutex // one refresh at a time
+	mu   sync.Mutex
+	data saved
+	// applied is what the ruleset in force holds (known after an apply).
+	applied map[string][]string
+	known   bool
+	read    time.Time
+	err     string
+	kick    chan struct{}
+	now     func() time.Time
+}
+
+func (r *Resolver) init() {
+	if r.data.Lookups == nil {
+		r.data.Lookups = map[string]*Lookup{}
+	}
+	if r.data.Nets == nil {
+		r.data.Nets = map[string][]netip.Prefix{}
+	}
+	if r.kick == nil {
+		r.kick = make(chan struct{}, 1)
+	}
+	if r.now == nil {
+		r.now = time.Now
+	}
+	if r.Log == nil {
+		r.Log = slog.Default()
+	}
+}
+
+// Restore reads what was saved before housecarld (or the machine)
+// stopped. Call it before the first apply.
+func (r *Resolver) Restore(ctx context.Context) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	if r.Load == nil {
+		return
+	}
+	v, err := r.Load(ctx)
+	if err != nil || v == "" {
+		return
+	}
+	var s saved
+	if err := json.Unmarshal([]byte(v), &s); err != nil {
+		r.Log.Warn("the saved Jarl set addresses can't be read; reading Jarl again", "err", err)
+		return
+	}
+	if s.Lookups != nil {
+		r.data.Lookups = s.Lookups
+	}
+	if s.Nets != nil {
+		r.data.Nets = s.Nets
+	}
+}
+
+// Kick has the next refresh happen now (after a change of the
+// configuration: a new name).
+func (r *Resolver) Kick() {
+	r.mu.Lock()
+	r.init()
+	k := r.kick
+	r.mu.Unlock()
+	select {
+	case k <- struct{}{}:
+	default:
+	}
+}
+
+// Run reads Jarl until ctx ends.
+func (r *Resolver) Run(ctx context.Context) {
+	r.mu.Lock()
+	r.init()
+	k := r.kick
+	every := r.Interval
+	r.mu.Unlock()
+	if every <= 0 {
+		every = time.Minute
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		r.Refresh(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		case <-k:
+		}
+	}
+}
+
+// Refresh reads Jarl now and puts what changed in force.
+func (r *Resolver) Refresh(ctx context.Context) {
+	r.run.Lock()
+	defer r.run.Unlock()
+	st := r.State()
+	r.read1(ctx, st)
+	r.sync(ctx, st)
+}
+
+// needed are the names the sets of st look up, by key.
+func needed(st *model.State) map[string]model.JarlName {
+	out := map[string]model.JarlName{}
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		for _, n := range s.Jarl.Names {
+			out[n.String()] = n
+		}
+	}
+	return out
+}
+
+// read1 reads every name from Jarl once.
+func (r *Resolver) read1(ctx context.Context, st *model.State) {
+	names := needed(st)
+	r.mu.Lock()
+	r.init()
+	src := r.Source
+	r.mu.Unlock()
+	// The zones' networks, for when they can't be seen (Entries).
+	r.rememberNets(st)
+	if src == nil || len(names) == 0 {
+		r.mu.Lock()
+		changed := false
+		for k := range r.data.Lookups {
+			if _, ok := names[k]; !ok {
+				delete(r.data.Lookups, k)
+				changed = true
+			}
+		}
+		r.mu.Unlock()
+		if changed {
+			r.save(ctx)
+		}
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	zones, err := src.AllZones(ctx)
+	now := r.now()
+	results := map[string]*Lookup{}
+	if err == nil {
+		l := &lookup{ctx: ctx, src: src, zones: zones, records: map[int64][]jarl.Record{}, errs: map[int64]error{}}
+		for k, n := range names {
+			results[k] = l.resolve(n)
+		}
+	}
+	r.mu.Lock()
+	if err != nil {
+		r.err = err.Error()
+	} else {
+		r.err, r.read = "", now
+	}
+	for k, n := range names {
+		old := r.data.Lookups[k]
+		res := results[k]
+		switch {
+		case res != nil && res.Error == "":
+			res.Checked = now
+			r.data.Lookups[k] = res
+		case old != nil:
+			// Keep what Jarl last said.
+			old.Error = errText(err, res)
+		default:
+			r.data.Lookups[k] = &Lookup{Name: n.Name, View: n.View, Error: errText(err, res)}
+		}
+		if err != nil || res != nil && res.Error != "" {
+			r.Log.Warn("reading a Jarl set's name failed; the set keeps the addresses it had", "name", n.String(), "err", r.data.Lookups[k].Error)
+		}
+	}
+	for k := range r.data.Lookups {
+		if _, ok := names[k]; !ok {
+			delete(r.data.Lookups, k)
+		}
+	}
+	r.mu.Unlock()
+	r.save(ctx)
+}
+
+func errText(err error, res *Lookup) string {
+	if err != nil {
+		return err.Error()
+	}
+	return res.Error
+}
+
+func (r *Resolver) save(ctx context.Context) {
+	if r.Save == nil {
+		return
+	}
+	r.mu.Lock()
+	b, err := json.Marshal(r.data)
+	r.mu.Unlock()
+	if err == nil {
+		err = r.Save(ctx, string(b))
+	}
+	if err != nil {
+		r.Log.Warn("saving the Jarl set addresses failed", "err", err)
+	}
+}
+
+// rememberNets keeps each zone's connected networks.
+func (r *Resolver) rememberNets(st *model.State) {
+	if r.Connected == nil {
+		return
+	}
+	now := zoneNets(r.Connected(st))
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for z, ps := range now {
+		r.data.Nets[z] = ps
+	}
+}
+
+func zoneNets(conn map[netip.Prefix]string) map[string][]netip.Prefix {
+	out := map[string][]netip.Prefix{}
+	for p, z := range conn {
+		out[z] = append(out[z], p)
+	}
+	for _, ps := range out {
+		sort.Slice(ps, func(i, j int) bool { return ps[i].String() < ps[j].String() })
+	}
+	return out
+}
+
+// sync applies the ruleset again if the sets' addresses changed.
+func (r *Resolver) sync(ctx context.Context, st *model.State) {
+	want := r.Entries(st)
+	r.mu.Lock()
+	known, applied := r.known, r.applied
+	r.mu.Unlock()
+	// No Jarl sets, nothing to do (and an empty configuration is never
+	// applied: its policies would close the machine).
+	if len(want) == 0 || st.Empty() || known && sameEntries(applied, want) || r.Apply == nil {
+		return
+	}
+	if err := r.Apply(ctx); err != nil {
+		r.Log.Error("putting the Jarl sets' new addresses in force failed", "err", err)
+		return
+	}
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		add, del := diff(applied[s.Name], want[s.Name])
+		if len(add) == 0 && len(del) == 0 {
+			continue
+		}
+		var parts []string
+		if len(add) > 0 {
+			parts = append(parts, "added "+strings.Join(add, ", "))
+		}
+		if len(del) > 0 {
+			parts = append(parts, "removed "+strings.Join(del, ", "))
+		}
+		detail := strings.Join(parts, "; ")
+		r.Log.Info("a Jarl set's addresses changed", "set", s.Name, "change", detail)
+		if r.Audit != nil {
+			r.Audit(ctx, s.Name, detail)
+		}
+	}
+}
+
+func sameEntries(a, b map[string][]string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for k, v := range a {
+		w, ok := b[k]
+		if !ok || strings.Join(v, ",") != strings.Join(w, ",") {
+			return false
+		}
+	}
+	return true
+}
+
+func diff(old, now []string) (add, del []string) {
+	was := map[string]bool{}
+	for _, a := range old {
+		was[a] = true
+	}
+	is := map[string]bool{}
+	for _, a := range now {
+		is[a] = true
+		if !was[a] {
+			add = append(add, a)
+		}
+	}
+	for _, a := range old {
+		if !is[a] {
+			del = append(del, a)
+		}
+	}
+	return add, del
+}
+
+// Applied implements nft.JarlSets.
+func (r *Resolver) Applied(entries map[string][]string) {
+	c := map[string][]string{}
+	for k, v := range entries {
+		c[k] = append([]string(nil), v...)
+	}
+	r.mu.Lock()
+	r.applied, r.known = c, true
+	r.mu.Unlock()
+}
+
+// Entries implements nft.JarlSets: every Jarl set of st (empty or not),
+// with the addresses its names have that lie within what it allows.
+func (r *Resolver) Entries(st *model.State) map[string][]string {
+	out := map[string][]string{}
+	nets := r.nets(st)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	for _, s := range st.Sets {
+		if s.Jarl == nil {
+			continue
+		}
+		seen := map[string]bool{}
+		list := []string{}
+		for _, n := range s.Jarl.Names {
+			lu := r.data.Lookups[n.String()]
+			if lu == nil {
+				continue
+			}
+			for _, a := range lu.Addresses {
+				if seen[a] || allowed(a, s.Jarl.Within, nets) != "" {
+					continue
+				}
+				seen[a] = true
+				list = append(list, a)
+			}
+		}
+		sortAddrs(list)
+		out[s.Name] = list
+	}
+	return out
+}
+
+// nets are the zones' connected networks: now, or as last seen.
+func (r *Resolver) nets(st *model.State) map[string][]netip.Prefix {
+	var now map[string][]netip.Prefix
+	if r.Connected != nil {
+		now = zoneNets(r.Connected(st))
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	out := map[string][]netip.Prefix{}
+	for _, z := range st.Zones {
+		if ps := now[z.Name]; len(ps) > 0 {
+			out[z.Name] = ps
+		} else {
+			out[z.Name] = r.data.Nets[z.Name]
+		}
+	}
+	return out
+}
+
+func sortAddrs(list []string) {
+	sort.Slice(list, func(i, j int) bool {
+		a, _ := netip.ParseAddr(list[i])
+		b, _ := netip.ParseAddr(list[j])
+		return a.Less(b)
+	})
+}
+
+// allowed says why an address isn't let in ("" if it is).
+func allowed(addr string, within []string, nets map[string][]netip.Prefix) string {
+	a, err := netip.ParseAddr(addr)
+	if err != nil {
+		return "not an address"
+	}
+	a = a.Unmap()
+	switch {
+	case a.IsUnspecified(), a.IsLoopback(), a.IsMulticast():
+		return "not a host's address (unspecified, loopback or multicast)"
+	}
+	for _, w := range within {
+		if l, err := model.ParseAddress(w); err == nil {
+			if l.Contains(a) {
+				return ""
+			}
+			continue
+		}
+		for _, p := range nets[w] {
+			if p.Contains(a) {
+				return ""
+			}
+		}
+	}
+	return "not within " + strings.Join(within, ", ")
+}
+
+// ---- looking names up
+
+// lookup reads Jarl's records for one refresh.
+type lookup struct {
+	ctx     context.Context
+	src     Source
+	zones   []jarl.ZoneInfo
+	records map[int64][]jarl.Record
+	errs    map[int64]error
+}
+
+func fqdn(n string) string {
+	n = strings.ToLower(strings.TrimSpace(n))
+	if !strings.HasSuffix(n, ".") {
+		n += "."
+	}
+	return n
+}
+
+func under(name, zone string) bool {
+	return zone == "." || name == zone || strings.HasSuffix(name, "."+zone)
+}
+
+func short(n string) string { return strings.TrimSuffix(n, ".") }
+
+// zoneFor finds the zone of Jarl's that answers for name in view ("":
+// the only view whose zones hold it). It returns the zone, the view, or
+// why there is none.
+func (l *lookup) zoneFor(name, view string) (*jarl.ZoneInfo, string, string) {
+	if view == "" {
+		views := map[string]bool{}
+		for _, z := range l.zones {
+			if z.Type != "hint" && under(name, fqdn(z.Name)) {
+				views[z.View] = true
+			}
+		}
+		switch len(views) {
+		case 0:
+			return nil, "", fmt.Sprintf("no zone in Jarl holds %s (or the token can't see it)", short(name))
+		case 1:
+			for v := range views {
+				view = v
+			}
+		default:
+			var vs []string
+			for v := range views {
+				vs = append(vs, v)
+			}
+			sort.Strings(vs)
+			return nil, "", fmt.Sprintf("%s is in Jarl's views %s: name the view, such as %s@%s", short(name), strings.Join(vs, ", "), short(name), vs[0])
+		}
+	}
+	var best *jarl.ZoneInfo
+	for i := range l.zones {
+		z := &l.zones[i]
+		zn := fqdn(z.Name)
+		if z.Type == "hint" || !strings.EqualFold(z.View, view) || !under(name, zn) {
+			continue
+		}
+		if best == nil || len(zn) > len(fqdn(best.Name)) {
+			best = z
+		}
+	}
+	where := ""
+	if view != "" {
+		where = " in view " + view
+	}
+	if best == nil {
+		return nil, view, fmt.Sprintf("no zone%s holds %s in Jarl (or the token can't see it)", where, short(name))
+	}
+	if best.Type == "in-view" {
+		var served *jarl.ZoneInfo
+		for i := range l.zones {
+			z := &l.zones[i]
+			if fqdn(z.Name) == fqdn(best.Name) && strings.EqualFold(z.View, best.InView) && z.Type != "in-view" {
+				served = z
+			}
+		}
+		if served == nil {
+			return nil, view, fmt.Sprintf("zone %s%s serves view %s's zone, which the token can't see", short(best.Name), where, best.InView)
+		}
+		best = served
+	}
+	if best.Disabled {
+		return nil, view, fmt.Sprintf("zone %s%s is disabled in Jarl", short(best.Name), where)
+	}
+	if best.Type != "primary" {
+		return nil, view, fmt.Sprintf("zone %s%s is a %s zone: Jarl doesn't hold its records", short(best.Name), where, best.Type)
+	}
+	return best, view, ""
+}
+
+func (l *lookup) zoneRecords(id int64) ([]jarl.Record, error) {
+	if err, ok := l.errs[id]; ok {
+		return nil, err
+	}
+	if rs, ok := l.records[id]; ok {
+		return rs, nil
+	}
+	rs, err := l.src.Records(l.ctx, id)
+	if err != nil {
+		l.errs[id] = err
+		return nil, err
+	}
+	l.records[id] = rs
+	return rs, nil
+}
+
+// resolve looks a name up as named would answer it in the view: its A
+// and AAAA records, or those of the names its CNAMEs lead to.
+func (l *lookup) resolve(n model.JarlName) *Lookup {
+	out := &Lookup{Name: n.Name, View: n.View}
+	name, view := fqdn(n.Name), n.View
+	seen := map[string]bool{name: true}
+	for hop := 0; ; hop++ {
+		z, v, problem := l.zoneFor(name, view)
+		if hop == 0 {
+			out.Found = v
+		}
+		if problem != "" {
+			if hop > 0 {
+				problem = fmt.Sprintf("its CNAME leads to %s, which Housecarl can't follow: %s (only names Jarl holds are followed)", short(name), problem)
+			}
+			out.Problem = problem
+			return out
+		}
+		view = v
+		recs, err := l.zoneRecords(z.ID)
+		if err != nil {
+			out.Error = err.Error()
+			return out
+		}
+		apex := fqdn(z.Name)
+		var addrs []string
+		cname := ""
+		for _, rec := range recs {
+			owner := fqdn(rec.Name)
+			switch {
+			case rec.Type == "NS" && owner != apex && under(name, owner):
+				out.Problem = fmt.Sprintf("%s is below a delegation (NS records at %s): Jarl doesn't hold its records", short(name), short(owner))
+				return out
+			case owner != name:
+			case rec.Type == "A" || rec.Type == "AAAA":
+				if a, err := netip.ParseAddr(strings.TrimSpace(rec.Data)); err == nil {
+					addrs = append(addrs, a.Unmap().String())
+				}
+			case rec.Type == "CNAME":
+				cname = target(rec.Data, apex)
+			}
+		}
+		if cname != "" {
+			if seen[cname] || len(out.Chain) >= maxChain {
+				out.Problem = fmt.Sprintf("its CNAMEs go round in a loop or are longer than %d (%s)", maxChain, strings.Join(append(out.Chain, short(cname)), " → "))
+				return out
+			}
+			seen[cname] = true
+			out.Chain = append(out.Chain, short(cname))
+			name = cname
+			continue
+		}
+		if len(addrs) == 0 {
+			if hop > 0 {
+				out.Problem = fmt.Sprintf("its CNAME leads to %s, which has no A or AAAA record in Jarl", short(name))
+			} else {
+				out.Problem = fmt.Sprintf("%s has no A, AAAA or CNAME record in Jarl (wildcards aren't followed)", short(name))
+			}
+			return out
+		}
+		sortAddrs(addrs)
+		out.Addresses = dedup(addrs)
+		return out
+	}
+}
+
+// target is a CNAME's target as an FQDN (a relative one is under the
+// zone, as in a zone file).
+func target(data, apex string) string {
+	t := strings.ToLower(strings.TrimSpace(data))
+	if strings.HasSuffix(t, ".") {
+		return t
+	}
+	if t == "@" {
+		return apex
+	}
+	if apex == "." {
+		return t + "."
+	}
+	return t + "." + apex
+}
+
+func dedup(in []string) []string {
+	var out []string
+	for i, a := range in {
+		if i == 0 || a != in[i-1] {
+			out = append(out, a)
+		}
+	}
+	return out
+}
+
+// ---- status
+
+// Status is what a Jarl set holds and why.
+func (r *Resolver) Status(st *model.State, set *model.AddressSet) api.JarlSet {
+	out := api.JarlSet{Set: set.Name, Entries: []string{}, Names: []api.JarlName{}, Within: []string{}}
+	if set.Jarl == nil {
+		return out
+	}
+	out.Within = append(out.Within, set.Jarl.Within...)
+	out.Entries = r.Entries(st)[set.Name]
+	nets := r.nets(st)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	out.Configured = r.Source != nil
+	if !r.read.IsZero() {
+		t := r.read
+		out.Read = &t
+	}
+	out.Error = r.err
+	for _, n := range set.Jarl.Names {
+		jn := api.JarlName{Name: n.Name, View: n.View, Addresses: []api.JarlAddress{}}
+		if lu := r.data.Lookups[n.String()]; lu != nil {
+			jn.Found, jn.Chain, jn.Problem, jn.Error = lu.Found, lu.Chain, lu.Problem, lu.Error
+			if !lu.Checked.IsZero() {
+				t := lu.Checked
+				jn.Checked = &t
+			}
+			for _, a := range lu.Addresses {
+				why := allowed(a, set.Jarl.Within, nets)
+				jn.Addresses = append(jn.Addresses, api.JarlAddress{Address: a, Used: why == "", Why: why})
+			}
+		}
+		out.Names = append(out.Names, jn)
+	}
+	return out
+}
+
+// Problems are the warnings about the Jarl sets for the status page.
+func (r *Resolver) Problems(st *model.State) []validate.Problem {
+	var out []validate.Problem
+	var sets []*model.AddressSet
+	for _, s := range st.Sets {
+		if s.Jarl != nil {
+			sets = append(sets, s)
+		}
+	}
+	if len(sets) == 0 {
+		return nil
+	}
+	r.mu.Lock()
+	configured, readErr := r.Source != nil, r.err
+	r.mu.Unlock()
+	if !configured {
+		out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_not_configured", Object: "jarl",
+			Message: "sets take their addresses from Jarl, but housecarl.yaml names no Jarl server (jarl.server, docs/jarl.md): they keep the addresses they last had, none if Jarl was never read"})
+	} else if readErr != "" {
+		out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_unreachable", Object: "jarl",
+			Message: "reading Jarl failed: " + readErr + "; the Jarl sets keep the addresses they had"})
+	}
+	for _, s := range sets {
+		obj := "set " + s.Name
+		for _, n := range r.Status(st, s).Names {
+			switch {
+			case n.Problem != "":
+				out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_name", Object: obj,
+					Message: n.Name + viewSuffix(n.View) + ": " + n.Problem + "; it adds no addresses"})
+			case n.Error != "" && readErr == "":
+				out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_unreachable", Object: obj,
+					Message: n.Name + viewSuffix(n.View) + ": reading it from Jarl failed: " + n.Error + "; it keeps the addresses it had"})
+			}
+			for _, a := range n.Addresses {
+				if !a.Used {
+					out = append(out, validate.Problem{Severity: validate.Warning, Code: "jarl_outside", Object: obj,
+						Message: fmt.Sprintf("%s%s has address %s, which is left out: %s", n.Name, viewSuffix(n.View), a.Address, a.Why)})
+				}
+			}
+		}
+	}
+	return out
+}
+
+func viewSuffix(v string) string {
+	if v == "" {
+		return ""
+	}
+	return "@" + v
+}

internal/jarlsets/jarlsets_test.go

@@ -0,0 +1,317 @@
+package jarlsets
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/netip"
+	"reflect"
+	"strings"
+	"testing"
+
+	"housecarl/internal/jarl"
+	"housecarl/internal/model"
+)
+
+// fakeJarl is a Jarl with a few zones.
+type fakeJarl struct {
+	zones   []jarl.ZoneInfo
+	records map[int64][]jarl.Record
+	down    bool           // AllZones fails
+	broken  map[int64]bool // Records fails
+	calls   int
+}
+
+func (f *fakeJarl) AllZones(ctx context.Context) ([]jarl.ZoneInfo, error) {
+	if f.down {
+		return nil, errors.New("can't reach Jarl")
+	}
+	return f.zones, nil
+}
+
+func (f *fakeJarl) Records(ctx context.Context, id int64) ([]jarl.Record, error) {
+	f.calls++
+	if f.broken[id] {
+		return nil, errors.New("Jarl answered 500")
+	}
+	return f.records[id], nil
+}
+
+func rec(name, typ, data string) jarl.Record { return jarl.Record{Name: name, Type: typ, Data: data} }
+
+// newFake: example.com in views internal (1) and external (2), linked
+// sub-zone wg.example.com in internal (3), other.org only in internal (4),
+// a secondary (5), a disabled zone (6), an in-view zone in vps (7).
+func newFake() *fakeJarl {
+	return &fakeJarl{
+		zones: []jarl.ZoneInfo{
+			{ID: 1, Name: "example.com.", View: "internal", Type: "primary"},
+			{ID: 2, Name: "example.com.", View: "external", Type: "primary"},
+			{ID: 3, Name: "wg.example.com.", View: "internal", Type: "primary"},
+			{ID: 4, Name: "other.org.", View: "internal", Type: "primary"},
+			{ID: 5, Name: "sec.net.", View: "internal", Type: "secondary"},
+			{ID: 6, Name: "off.net.", View: "internal", Type: "primary", Disabled: true},
+			{ID: 7, Name: "other.org.", View: "vps", Type: "in-view", InView: "internal"},
+			{ID: 8, Name: ".", View: "internal", Type: "hint"},
+		},
+		records: map[int64][]jarl.Record{
+			1: {
+				rec("example.com.", "NS", "ns1.example.com."),
+				rec("nas.example.com.", "A", "10.10.0.20"),
+				rec("nas.example.com.", "A", "10.10.0.21"),
+				rec("nas.example.com.", "AAAA", "2001:db8:10::20"),
+				rec("files.example.com.", "CNAME", "nas.example.com."),
+				rec("vpn.example.com.", "CNAME", "peer1.wg.example.com."),
+				rec("ext.example.com.", "CNAME", "host.duckdns.org."),
+				rec("loop1.example.com.", "CNAME", "loop2.example.com."),
+				rec("loop2.example.com.", "CNAME", "loop1"),
+				rec("rel.example.com.", "CNAME", "nas"),
+				rec("mail.example.com.", "MX", "10 nas.example.com."),
+				rec("lab.example.com.", "NS", "ns.lab.example.com."),
+				rec("www.example.com.", "A", "10.10.0.30"),
+				rec("bad.example.com.", "A", "127.0.0.1"),
+				rec("far.example.com.", "A", "192.0.2.9"),
+				rec("dead.example.com.", "CNAME", "nothing.example.com."),
+				rec("toother.example.com.", "CNAME", "srv.other.org."),
+			},
+			2: {
+				rec("www.example.com.", "A", "203.0.113.30"),
+			},
+			3: {
+				rec("peer1.wg.example.com.", "A", "10.50.50.2"),
+			},
+			4: {
+				rec("srv.other.org.", "A", "10.10.0.40"),
+			},
+		},
+		broken: map[int64]bool{},
+	}
+}
+
+func resolveOne(t *testing.T, f *fakeJarl, s string) *Lookup {
+	t.Helper()
+	l := &lookup{ctx: context.Background(), src: f, zones: f.zones, records: map[int64][]jarl.Record{}, errs: map[int64]error{}}
+	return l.resolve(model.ParseJarlName(s))
+}
+
+func TestResolve(t *testing.T) {
+	f := newFake()
+	cases := []struct {
+		name    string
+		addrs   []string
+		chain   []string
+		problem string // part of it
+		found   string
+	}{
+		{name: "nas.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, found: "internal"},
+		{name: "NAS.Example.com.@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, found: "internal"},
+		{name: "files.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, chain: []string{"nas.example.com"}},
+		{name: "rel.example.com@internal", addrs: []string{"10.10.0.20", "10.10.0.21", "2001:db8:10::20"}, chain: []string{"nas.example.com"}},
+		// Into the sub-zone, and into another zone of the same view.
+		{name: "vpn.example.com@internal", addrs: []string{"10.50.50.2"}, chain: []string{"peer1.wg.example.com"}},
+		{name: "toother.example.com@internal", addrs: []string{"10.10.0.40"}, chain: []string{"srv.other.org"}},
+		// Only internal has wg.example.com, but external's example.com
+		// answers for the name too (that it doesn't exist): the view must be
+		// named.
+		{name: "peer1.wg.example.com", problem: "views external, internal"},
+		{name: "srv.other.org", problem: "views internal, vps"},
+		// Views give different answers.
+		{name: "www.example.com@internal", addrs: []string{"10.10.0.30"}},
+		{name: "www.example.com@external", addrs: []string{"203.0.113.30"}},
+		{name: "www.example.com", problem: "views external, internal: name the view"},
+		{name: "www.example.com@vps", problem: "no zone in view vps holds www.example.com"},
+		// An in-view zone serves the other view's records.
+		{name: "srv.other.org@vps", addrs: []string{"10.10.0.40"}},
+		{name: "ext.example.com@internal", chain: []string{"host.duckdns.org"}, problem: "its CNAME leads to host.duckdns.org, which Housecarl can't follow"},
+		{name: "loop1.example.com@internal", problem: "loop"},
+		{name: "dead.example.com@internal", chain: []string{"nothing.example.com"}, problem: "nothing.example.com, which has no A or AAAA record"},
+		{name: "mail.example.com@internal", problem: "has no A, AAAA or CNAME record"},
+		{name: "missing.example.com@internal", problem: "has no A, AAAA or CNAME record"},
+		{name: "host.lab.example.com@internal", problem: "below a delegation (NS records at lab.example.com)"},
+		{name: "a.sec.net", problem: "is a secondary zone"},
+		{name: "a.off.net", problem: "is disabled"},
+		{name: "nowhere.test", problem: "no zone in Jarl holds nowhere.test"},
+	}
+	for _, c := range cases {
+		lu := resolveOne(t, f, c.name)
+		if lu.Error != "" {
+			t.Errorf("%s: error %s", c.name, lu.Error)
+		}
+		if !reflect.DeepEqual(lu.Addresses, c.addrs) {
+			t.Errorf("%s: addresses %v, want %v (problem %q)", c.name, lu.Addresses, c.addrs, lu.Problem)
+		}
+		if c.chain != nil && !reflect.DeepEqual(lu.Chain, c.chain) {
+			t.Errorf("%s: chain %v, want %v", c.name, lu.Chain, c.chain)
+		}
+		if c.problem == "" && lu.Problem != "" || !strings.Contains(lu.Problem, c.problem) {
+			t.Errorf("%s: problem %q, want %q", c.name, lu.Problem, c.problem)
+		}
+		if c.found != "" && lu.Found != c.found {
+			t.Errorf("%s: found in %q, want %q", c.name, lu.Found, c.found)
+		}
+	}
+}
+
+func TestResolveNoViews(t *testing.T) {
+	f := &fakeJarl{zones: []jarl.ZoneInfo{{ID: 1, Name: "lan.", Type: "primary"}},
+		records: map[int64][]jarl.Record{1: {rec("nas.lan.", "A", "192.168.1.5")}}}
+	if lu := resolveOne(t, f, "nas.lan"); !reflect.DeepEqual(lu.Addresses, []string{"192.168.1.5"}) || lu.Problem != "" {
+		t.Errorf("%+v", lu)
+	}
+}
+
+// harness is a resolver over the fake Jarl, with lan = 10.10.0.0/24 and
+// wg = 10.50.50.0/24 connected.
+type harness struct {
+	r       *Resolver
+	f       *fakeJarl
+	st      *model.State
+	conn    map[netip.Prefix]string
+	applies int
+	audits  []string
+	saved   string
+}
+
+func newHarness(sets ...*model.AddressSet) *harness {
+	h := &harness{f: newFake(), conn: map[netip.Prefix]string{
+		netip.MustParsePrefix("10.10.0.0/24"):  "lan",
+		netip.MustParsePrefix("10.50.50.0/24"): "wg",
+	}}
+	h.st = &model.State{Zones: []*model.Zone{{Name: "lan"}, {Name: "wg"}, {Name: "wan"}}, Sets: sets}
+	h.r = &Resolver{Source: h.f, State: func() *model.State { return h.st }, Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
+		Connected: func(*model.State) map[netip.Prefix]string { return h.conn },
+		Apply: func(ctx context.Context) error {
+			h.applies++
+			h.r.Applied(h.r.Entries(h.st))
+			return nil
+		},
+		Audit: func(ctx context.Context, set, detail string) { h.audits = append(h.audits, set+": "+detail) },
+		Load:  func(ctx context.Context) (string, error) { return h.saved, nil },
+		Save:  func(ctx context.Context, v string) error { h.saved = v; return nil }}
+	return h
+}
+
+func jset(name string, within []string, names ...string) *model.AddressSet {
+	j := &model.JarlSource{Within: within}
+	for _, n := range names {
+		j.Names = append(j.Names, model.ParseJarlName(n))
+	}
+	return &model.AddressSet{Name: name, Jarl: j}
+}
+
+func TestWithin(t *testing.T) {
+	h := newHarness(
+		jset("nas", []string{"lan"}, "nas.example.com@internal", "vpn.example.com@internal", "bad.example.com@internal", "far.example.com@internal"),
+		jset("wide", []string{"lan", "wg", "192.0.2.0/24", "2001:db8:10::/64"}, "nas.example.com@internal", "vpn.example.com@internal", "far.example.com@internal"),
+		jset("none", []string{"lan"}, "missing.example.com@internal"),
+	)
+	h.r.Refresh(context.Background())
+	got := h.r.Entries(h.st)
+	want := map[string][]string{
+		"nas":  {"10.10.0.20", "10.10.0.21"},
+		"wide": {"10.10.0.20", "10.10.0.21", "10.50.50.2", "192.0.2.9", "2001:db8:10::20"},
+		"none": {},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("entries %v, want %v", got, want)
+	}
+	st := h.r.Status(h.st, h.st.Sets[0])
+	why := map[string]string{}
+	for _, n := range st.Names {
+		for _, a := range n.Addresses {
+			why[a.Address] = a.Why
+		}
+	}
+	if why["10.10.0.20"] != "" || !strings.Contains(why["10.50.50.2"], "not within lan") || !strings.Contains(why["127.0.0.1"], "loopback") || why["2001:db8:10::20"] == "" {
+		t.Errorf("why: %v", why)
+	}
+	codes := map[string]int{}
+	for _, p := range h.r.Problems(h.st) {
+		codes[p.Code]++
+	}
+	// nas: 3 left out (v6, wg, loopback, far = 4); wide: none; none: a name problem.
+	if codes["jarl_outside"] != 4 || codes["jarl_name"] != 1 || codes["jarl_unreachable"] != 0 {
+		t.Errorf("problems %v: %+v", codes, h.r.Problems(h.st))
+	}
+	// The lan interface loses its address: its networks as last seen count.
+	h.conn = map[netip.Prefix]string{}
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.20", "10.10.0.21"}) {
+		t.Errorf("without lan's address: %v", got)
+	}
+}
+
+func TestKeepsWhatItHad(t *testing.T) {
+	h := newHarness(jset("nas", []string{"lan", "wg"}, "nas.example.com@internal", "peer1.wg.example.com@internal"))
+	ctx := context.Background()
+	h.r.Refresh(ctx)
+	if h.applies != 1 || len(h.audits) != 1 || !strings.Contains(h.audits[0], "added 10.10.0.20, 10.10.0.21, 10.50.50.2") {
+		t.Fatalf("first read: %d applies, audits %v", h.applies, h.audits)
+	}
+	// Nothing changed: no apply.
+	h.r.Refresh(ctx)
+	if h.applies != 1 {
+		t.Errorf("unchanged: %d applies", h.applies)
+	}
+	// Jarl down: the addresses stay, a warning says so.
+	h.f.down = true
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; len(got) != 3 || h.applies != 1 {
+		t.Errorf("Jarl down: %v, %d applies", got, h.applies)
+	}
+	if ps := h.r.Problems(h.st); len(ps) != 2 || ps[0].Code != "jarl_unreachable" || ps[1].Code != "jarl_outside" {
+		t.Errorf("Jarl down: %+v", ps)
+	}
+	// One zone's records fail: only its name keeps the old addresses.
+	h.f.down = false
+	h.f.broken[3] = true
+	h.f.records[1][1] = rec("nas.example.com.", "A", "10.10.0.22")
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22", "10.50.50.2"}) {
+		t.Errorf("one zone down: %v", got)
+	}
+	if h.applies != 2 || !strings.Contains(h.audits[1], "added 10.10.0.22; removed 10.10.0.20") {
+		t.Errorf("audits %v", h.audits)
+	}
+	// After a restart, before Jarl is read: what was saved.
+	r2 := &Resolver{Load: h.r.Load, State: h.r.State, Connected: h.r.Connected}
+	r2.Restore(ctx)
+	if got := r2.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22", "10.50.50.2"}) {
+		t.Errorf("restored: %v", got)
+	}
+	// Jarl answers that the name is gone: its addresses go.
+	h.f.broken[3] = false
+	h.f.records[3] = nil
+	h.r.Refresh(ctx)
+	if got := h.r.Entries(h.st)["nas"]; !reflect.DeepEqual(got, []string{"10.10.0.21", "10.10.0.22"}) {
+		t.Errorf("deleted record: %v", got)
+	}
+	// A name taken out of the set is forgotten.
+	h.st.Sets[0].Jarl.Names = h.st.Sets[0].Jarl.Names[:1]
+	h.r.Refresh(ctx)
+	if len(h.r.data.Lookups) != 1 {
+		t.Errorf("lookups %v", h.r.data.Lookups)
+	}
+}
+
+func TestNotConfigured(t *testing.T) {
+	h := newHarness(jset("nas", []string{"lan"}, "nas.example.com@internal"))
+	h.r.Source = nil
+	h.r.Refresh(context.Background())
+	if ps := h.r.Problems(h.st); len(ps) != 1 || ps[0].Code != "jarl_not_configured" {
+		t.Errorf("%+v", ps)
+	}
+	if got := h.r.Entries(h.st); !reflect.DeepEqual(got, map[string][]string{"nas": {}}) {
+		t.Errorf("%v", got)
+	}
+}
+
+// TestNoSetsNoApply: without Jarl sets nothing is applied, not even at the
+// first read (an empty configuration must never be applied).
+func TestNoSetsNoApply(t *testing.T) {
+	h := newHarness()
+	h.r.Refresh(context.Background())
+	if h.applies != 0 {
+		t.Errorf("%d applies", h.applies)
+	}
+}

internal/linkwatch/linkwatch.go

@@ -0,0 +1,96 @@
+// Package linkwatch tells when the machine's network interfaces change:
+// one appears, goes, or comes back (ppp0 when PPPoE reconnects). It
+// listens to the kernel's rtnetlink link messages and calls back once
+// they have been quiet for a moment, since one change brings several.
+package linkwatch
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Settle is how long the link messages must be quiet before the callback.
+const Settle = 2 * time.Second
+
+// Watch calls changed after interfaces changed, until ctx ends. It returns
+// an error only if it can't listen at all.
+func Watch(ctx context.Context, changed func()) error {
+	fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC|unix.SOCK_NONBLOCK, unix.NETLINK_ROUTE)
+	if err != nil {
+		return fmt.Errorf("opening a netlink socket: %w", err)
+	}
+	if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK, Groups: unix.RTMGRP_LINK}); err != nil {
+		unix.Close(fd)
+		return fmt.Errorf("listening to link changes: %w", err)
+	}
+	// A non-blocking descriptor goes through Go's poller, so closing the
+	// file ends a Read in progress.
+	f := os.NewFile(uintptr(fd), "rtnetlink")
+	go func() {
+		<-ctx.Done()
+		f.Close()
+	}()
+	events := make(chan struct{}, 1)
+	go func() {
+		buf := make([]byte, 64<<10)
+		for {
+			n, err := f.Read(buf)
+			switch {
+			case err != nil && ctx.Err() != nil:
+				close(events)
+				return
+			case errors.Is(err, unix.ENOBUFS):
+				// Messages were lost: something changed anyway.
+			case err != nil:
+				time.Sleep(time.Second)
+				continue
+			case !links(buf[:n]):
+				continue
+			}
+			select {
+			case events <- struct{}{}:
+			default:
+			}
+		}
+	}()
+	var timer <-chan time.Time
+	for {
+		select {
+		case _, ok := <-events:
+			if !ok {
+				return nil
+			}
+			timer = time.After(Settle)
+		case <-timer:
+			timer = nil
+			changed()
+		}
+	}
+}
+
+// links reports whether a netlink datagram holds a link being added,
+// changed or deleted.
+func links(b []byte) bool {
+	for len(b) >= unix.NLMSG_HDRLEN {
+		size := int(binary.NativeEndian.Uint32(b[0:4]))
+		typ := binary.NativeEndian.Uint16(b[4:6])
+		if size < unix.NLMSG_HDRLEN || size > len(b) {
+			return false
+		}
+		if typ == unix.RTM_NEWLINK || typ == unix.RTM_DELLINK {
+			return true
+		}
+		size = (size + unix.NLMSG_ALIGNTO - 1) &^ (unix.NLMSG_ALIGNTO - 1)
+		if size >= len(b) {
+			return false
+		}
+		b = b[size:]
+	}
+	return false
+}

internal/linkwatch/linkwatch_test.go

@@ -0,0 +1,36 @@
+package linkwatch
+
+import (
+	"encoding/binary"
+	"testing"
+
+	"golang.org/x/sys/unix"
+)
+
+// msg is a netlink message of a type, with a payload of n bytes.
+func msg(typ uint16, n int) []byte {
+	b := make([]byte, unix.NLMSG_HDRLEN+n)
+	binary.NativeEndian.PutUint32(b[0:4], uint32(len(b)))
+	binary.NativeEndian.PutUint16(b[4:6], typ)
+	return b
+}
+
+func TestLinks(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		b    []byte
+		want bool
+	}{
+		{"a new link", msg(unix.RTM_NEWLINK, 16), true},
+		{"a link going", msg(unix.RTM_DELLINK, 16), true},
+		{"an address", msg(unix.RTM_NEWADDR, 16), false},
+		{"a link after an address (padded)", append(msg(unix.RTM_NEWADDR, 13), append(make([]byte, 3), msg(unix.RTM_NEWLINK, 8)...)...), true},
+		{"truncated", msg(unix.RTM_NEWLINK, 16)[:10], false},
+		{"a length beyond the datagram", msg(unix.RTM_NEWLINK, 16)[:unix.NLMSG_HDRLEN+4], false},
+		{"empty", nil, false},
+	} {
+		if got := links(c.b); got != c.want {
+			t.Errorf("%s: %v", c.name, got)
+		}
+	}
+}

internal/matrix/matrix.go

@@ -0,0 +1,546 @@
+// Package matrix works out what the rules do to new connections between
+// zones, for an overview: for each pair it walks the rules as nftables
+// would, in order. A rule that applies to the whole pair decides it; one
+// that applies to some of its traffic (some addresses, services, times,
+// rates…) is an exception, and the walk goes on; what is left falls to
+// the chain's policy.
+package matrix
+
+import (
+	"fmt"
+	"net/netip"
+	"regexp"
+	"strings"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+// Firewall stands for the machine itself in the matrix.
+const Firewall = "firewall"
+
+// Options tell Build about the machine.
+type Options struct {
+	// Connected maps the networks directly connected to the firewall's
+	// interfaces to their zones. A rule whose addresses all lie in other
+	// zones' networks can't apply to a pair, and is left out of it.
+	Connected map[netip.Prefix]string
+}
+
+// Build works out the matrix of a configuration.
+func Build(st *model.State, opts ...Options) api.Matrix {
+	var o Options
+	if len(opts) > 0 {
+		o = opts[0]
+	}
+	m := api.Matrix{Zones: []string{}, Cells: []api.MatrixCell{}}
+	for _, z := range st.Zones {
+		m.Zones = append(m.Zones, z.Name)
+	}
+	for _, from := range m.Zones {
+		for _, to := range m.Zones {
+			m.Cells = append(m.Cells, cell(st, o, model.ChainForward, from, to))
+		}
+		m.Cells = append(m.Cells, cell(st, o, model.ChainInput, from, Firewall))
+	}
+	for _, to := range m.Zones {
+		m.Cells = append(m.Cells, cell(st, o, model.ChainOutput, Firewall, to))
+	}
+	return m
+}
+
+// Cell returns one pair (from or to may be Firewall).
+func Cell(st *model.State, o Options, from, to string) (api.MatrixCell, bool) {
+	known := func(z string) bool { return st.Zone(z) != nil }
+	switch {
+	case from == Firewall && known(to):
+		return cell(st, o, model.ChainOutput, from, to), true
+	case to == Firewall && known(from):
+		return cell(st, o, model.ChainInput, from, to), true
+	case known(from) && known(to):
+		return cell(st, o, model.ChainForward, from, to), true
+	}
+	return api.MatrixCell{}, false
+}
+
+func cell(st *model.State, o Options, chain, from, to string) api.MatrixCell {
+	c := walkCell(st, o, chain, from, to)
+	sum(&c)
+	if from == to && chain == model.ChainForward {
+		c.SameZone, c.Note = true, sameZoneNote(st.Zone(from))
+	}
+	return c
+}
+
+// sum fills in the overview: the summary, the exceptions counted by
+// verdict, and what the allowing ones let through.
+func sum(c *api.MatrixCell) {
+	c.Open = []string{}
+	seen := map[string]bool{}
+	for _, e := range c.Exceptions {
+		switch e.Verdict {
+		case "allowed":
+			c.Allowed++
+			if !seen[e.Short] {
+				seen[e.Short] = true
+				c.Open = append(c.Open, e.Short)
+			}
+		case "blocked":
+			c.Blocked++
+		}
+	}
+	c.Summary = c.Verdict
+	if c.Verdict == "blocked" && c.Allowed > 0 || c.Verdict == "allowed" && c.Blocked > 0 {
+		c.Summary = "partly"
+	}
+}
+
+// sameZoneNote says what of a zone's traffic to itself the firewall sees:
+// only what it routes between the zone's interfaces.
+func sameZoneNote(z *model.Zone) string {
+	if z == nil {
+		return ""
+	}
+	if len(z.Interfaces) == 1 && !strings.HasSuffix(z.Interfaces[0], "*") {
+		return fmt.Sprintf("traffic within %s stays on its own network: it passes the firewall only if routed back out of %s", z.Name, z.Interfaces[0])
+	}
+	return fmt.Sprintf("only traffic routed between %s's interfaces (%s) passes the firewall; within one of them it stays on its own network", z.Name, strings.Join(z.Interfaces, ", "))
+}
+
+func walkCell(st *model.State, o Options, chain, from, to string) api.MatrixCell {
+	c := api.MatrixCell{From: from, To: to, Chain: chain, Exceptions: []api.MatrixException{}}
+	b := st.Settings.Builtins
+	fromZone := st.Zone(from)
+	external := fromZone != nil && fromZone.External
+	// What comes in passes the early chain first.
+	if chain != model.ChainOutput {
+		if b.Antispoof && external {
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: model.ChainEarly, Label: "built-in: antispoof", Verdict: "blocked",
+				What: "sources that can't come from outside (private ranges, your own prefixes)", Short: "spoofed sources", Marks: []string{"b-antispoof", "b-antispoof6"}})
+		}
+		if decided := walk(st, o, &c, model.ChainEarly, from, ""); decided {
+			return c
+		}
+	}
+	// The built-ins that let some new connections in.
+	switch chain {
+	case model.ChainInput:
+		if b.ICMPv6 {
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: chain, Label: "built-in: icmpv6", Verdict: "allowed", What: "ICMPv6 errors, echo, neighbour discovery, MLD",
+				Short: "ICMPv6", Marks: []string{"b-icmpv6", "b-icmpv6-nd", "b-icmpv6-mld"}})
+		}
+		if b.PingInternal && !external {
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: chain, Label: "built-in: ping", Verdict: "allowed", What: "ping (ICMP echo)", Short: "ping", Marks: []string{"b-ping"}})
+		}
+		if b.DHCPv6Client && external {
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: chain, Label: "built-in: dhcpv6-client", Verdict: "allowed", What: "DHCPv6 replies to the firewall",
+				Short: "DHCPv6", Marks: []string{"b-dhcpv6-client"}})
+		}
+	case model.ChainForward:
+		if marks := forwards(st); len(marks) > 0 && external && !st.Zone(to).External {
+			n := fmt.Sprintf("%d port forward%s", len(marks), map[bool]string{true: "", false: "s"}[len(marks) == 1])
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: chain, Label: "port forwards", Verdict: "allowed",
+				What: n + " to inside hosts", Short: n, Marks: marks})
+		}
+	case model.ChainOutput:
+		if st.Settings.OutputPolicy == model.PolicyDrop && b.ICMPv6 {
+			c.Exceptions = append(c.Exceptions, api.MatrixException{Chain: chain, Label: "built-in: icmpv6", Verdict: "allowed", What: "ICMPv6 errors, echo, neighbour discovery, MLD",
+				Short: "ICMPv6", Marks: []string{"b-icmpv6"}})
+		}
+	}
+	if walk(st, o, &c, chain, from, to) {
+		return c
+	}
+	policy := map[string]string{model.ChainInput: st.Settings.InputPolicy, model.ChainForward: st.Settings.ForwardPolicy, model.ChainOutput: st.Settings.OutputPolicy}[chain]
+	c.Verdict, c.DecidedBy = verdictOf(policy), chain+" policy"
+	if policy == model.PolicyDrop {
+		c.Marks = []string{"p-" + chain} // only a drop policy is counted
+	}
+	return c
+}
+
+// forwards returns the counters' marks of the port forwards that send
+// connections inside.
+func forwards(st *model.State) []string {
+	var marks []string
+	for _, f := range st.Forwards {
+		if !f.Disabled && f.To != "" {
+			marks = append(marks, fmt.Sprintf("f%d", f.ID))
+		}
+	}
+	return marks
+}
+
+// walk goes through a chain's rules for the pair, as nftables checks
+// them (those that check every packet first). It reports whether a rule
+// decided the whole pair.
+func walk(st *model.State, o Options, c *api.MatrixCell, chain, from, to string) bool {
+	var rules []*model.Rule
+	for _, every := range []bool{true, false} {
+		for _, r := range st.Rules {
+			if r.Chain == chain && !r.Disabled && r.ChecksEveryPacket() == every {
+				rules = append(rules, r)
+			}
+		}
+	}
+	for _, r := range rules {
+		mark := []string{fmt.Sprintf("r%d", r.ID)}
+		if r.Raw != "" {
+			rr := readRaw(r.Raw)
+			switch {
+			case !rr.ok:
+				c.Exceptions = append(c.Exceptions, api.MatrixException{RuleID: r.ID, Chain: chain, Label: "rule " + r.Label(), Verdict: "unknown", What: "a raw rule: " + r.Raw,
+					Short: "a raw rule", Marks: mark})
+			case !rr.forNew || chain == model.ChainEarly && rr.verdict == "allowed":
+				// Not for new connections; an early accept decides nothing.
+			case rr.match == "":
+				c.Verdict, c.DecidedBy, c.RuleID, c.Marks = rr.verdict, "rule "+r.Label(), r.ID, mark
+				return true
+			default:
+				c.Exceptions = append(c.Exceptions, api.MatrixException{RuleID: r.ID, Chain: chain, Label: "rule " + r.Label(), Verdict: rr.verdict, What: "raw: " + rr.match,
+					Short: rr.short, Marks: mark})
+			}
+			continue
+		}
+		if !zones(r, from, to) || !forNew(r) {
+			continue
+		}
+		// Addresses that all live behind other zones can't come from (or
+		// go to) this pair's.
+		if !r.NotSources && elsewhere(st, o, r.Sources, from) || !r.NotDests && to != Firewall && to != "" && elsewhere(st, o, r.Destinations, to) {
+			continue
+		}
+		if chain == model.ChainEarly && r.Action != model.ActionDrop && r.Action != model.ActionReject {
+			continue // an early accept decides nothing: input or forward still check
+		}
+		v := verdictOf(r.Action)
+		what := conditions(st, r)
+		if r.Action == model.ActionLog || what != "" {
+			if what == "" {
+				what = "everything"
+			}
+			c.Exceptions = append(c.Exceptions, api.MatrixException{RuleID: r.ID, Chain: chain, Label: "rule " + r.Label(), Verdict: v, What: what,
+				Short: short(r, what), Marks: mark})
+			continue
+		}
+		c.Verdict, c.DecidedBy, c.RuleID, c.Marks = v, "rule "+r.Label(), r.ID, mark
+		return true
+	}
+	return false
+}
+
+func verdictOf(action string) string {
+	switch action {
+	case model.ActionAccept: // and the accept policy
+		return "allowed"
+	case model.ActionLog:
+		return "logged"
+	}
+	return "blocked"
+}
+
+// elsewhere reports whether every address of refs (addresses, sets) lies
+// in networks connected to zones other than zone: then traffic of zone
+// can't have them. An empty list, a dynamic or Jarl set, or an address the
+// firewall isn't directly connected to (a routed network) might.
+func elsewhere(st *model.State, o Options, refs []string, zone string) bool {
+	if len(refs) == 0 || len(o.Connected) == 0 || zone == Firewall {
+		return false
+	}
+	var entries []string
+	for _, r := range refs {
+		if s := st.Set(r); s != nil {
+			if s.Runtime() {
+				return false
+			}
+			entries = append(entries, s.Entries...)
+		} else {
+			entries = append(entries, r)
+		}
+	}
+	if len(entries) == 0 {
+		return false
+	}
+	for _, e := range entries {
+		a, err := model.ParseAddress(e)
+		if err != nil {
+			return false
+		}
+		first, last := a.Prefix.Masked().Addr(), lastAddr(a.Prefix)
+		if a.First.IsValid() {
+			first, last = a.First, a.End
+		}
+		owner := ""
+		for net, z := range o.Connected {
+			if net.Contains(first) && net.Contains(last) {
+				owner = z
+				break
+			}
+		}
+		if owner == "" || strings.EqualFold(owner, zone) {
+			return false
+		}
+	}
+	return true
+}
+
+// lastAddr is a prefix's last address.
+func lastAddr(p netip.Prefix) netip.Addr {
+	b := p.Masked().Addr().AsSlice()
+	for i := p.Bits(); i < len(b)*8; i++ {
+		b[i/8] |= 0x80 >> (i % 8)
+	}
+	a, _ := netip.AddrFromSlice(b)
+	return a
+}
+
+// zones reports whether a rule's zones take in the pair.
+func zones(r *model.Rule, from, to string) bool {
+	in := func(list []string, not bool, z string) bool {
+		if len(list) == 0 {
+			return true
+		}
+		for _, x := range list {
+			if strings.EqualFold(x, z) {
+				return !not
+			}
+		}
+		return not
+	}
+	if from == Firewall {
+		if len(r.InZones) > 0 {
+			return false
+		}
+	} else if !in(r.InZones, r.NotIn, from) {
+		return false
+	}
+	if to == "" || to == Firewall {
+		return len(r.OutZones) == 0
+	}
+	return in(r.OutZones, r.NotOut, to)
+}
+
+// forNew reports whether a rule can match a new connection's first
+// packet (its connection states allow "new").
+func forNew(r *model.Rule) bool {
+	if len(r.CtState) == 0 {
+		return true
+	}
+	for _, s := range r.CtState {
+		if s == "new" {
+			return true
+		}
+	}
+	return false
+}
+
+// raw is what the matrix can read of a raw rule.
+type raw struct {
+	ok      bool   // read: it applies to every pair of its chain
+	forNew  bool   // it can match a new connection
+	verdict string // allowed or blocked
+	match   string // what it matches ("" for everything)
+	short   string
+}
+
+// rawStops are words that make a raw rule's reach or effect unclear to
+// the matrix: interfaces (which zones?), jumps, NAT, changes, verdict maps.
+var rawStops = map[string]bool{
+	"iifname": true, "oifname": true, "iif": true, "oif": true, "iiftype": true, "oiftype": true,
+	"jump": true, "goto": true, "return": true, "vmap": true, "queue": true, "continue": true,
+	"dnat": true, "snat": true, "masquerade": true, "redirect": true, "tproxy": true,
+	"set": true, "notrack": true, "dup": true, "fwd": true, "synproxy": true, "flow": true,
+}
+
+// readRaw reads a raw rule that is a match and then accept, drop or
+// reject: the verdict is the last word (but a reject's "with …"); counter
+// and log only count and log.
+func readRaw(text string) raw {
+	toks := fields(text)
+	for i, t := range toks {
+		if t == "comment" {
+			toks = toks[:i]
+			break
+		}
+	}
+	verdict := -1
+	for i, t := range toks {
+		if strings.HasPrefix(t, `"`) {
+			continue
+		}
+		w := strings.Trim(t, ",{}")
+		if rawStops[w] {
+			return raw{}
+		}
+		if w == "accept" || w == "drop" || w == "reject" {
+			verdict = i
+		}
+	}
+	if verdict < 0 || verdict != len(toks)-1 && !(toks[verdict] == "reject" && toks[verdict+1] == "with") {
+		return raw{}
+	}
+	rr := raw{ok: true, forNew: true, verdict: verdictOf(toks[verdict])}
+	var match []string
+	for i := 0; i < verdict; i++ {
+		switch toks[i] {
+		case "counter":
+			continue
+		case "log":
+			// Its options run up to the next match.
+			for i+1 < verdict && map[string]bool{"prefix": true, "level": true, "group": true, "flags": true}[toks[i+1]] {
+				i += 2
+			}
+			continue
+		}
+		match = append(match, toks[i])
+	}
+	rr.match = strings.Join(match, " ")
+	// ct state without new: not for new connections. The states are one
+	// word ("established,related") or a set ("{ established, related }").
+	for i := 0; i+2 < len(match); i++ {
+		if match[i] != "ct" || match[i+1] != "state" {
+			continue
+		}
+		list := match[i+2]
+		if list == "{" || strings.HasPrefix(list, "{") {
+			var b []string
+			for j := i + 2; j < len(match); j++ {
+				b = append(b, match[j])
+				if strings.HasSuffix(match[j], "}") {
+					break
+				}
+			}
+			list = strings.Join(b, " ")
+		}
+		rr.forNew = false
+		for _, st := range strings.FieldsFunc(list, func(c rune) bool { return c == ',' || c == ' ' || c == '{' || c == '}' }) {
+			rr.forNew = rr.forNew || st == "new"
+		}
+	}
+	rr.short = rr.match
+	if m := icmpRe.FindStringSubmatch(rr.match); m != nil {
+		rr.short = m[1]
+	}
+	if len(rr.short) > 40 {
+		rr.short = rr.short[:39] + "…"
+	}
+	return rr
+}
+
+// fields splits a rule into words, a quoted string being one.
+func fields(s string) []string {
+	var out []string
+	var cur strings.Builder
+	quoted := false
+	for _, c := range s {
+		switch {
+		case c == '"':
+			quoted = !quoted
+			cur.WriteRune(c)
+		case !quoted && (c == ' ' || c == '\t'):
+			if cur.Len() > 0 {
+				out = append(out, cur.String())
+				cur.Reset()
+			}
+		default:
+			cur.WriteRune(c)
+		}
+	}
+	if cur.Len() > 0 {
+		out = append(out, cur.String())
+	}
+	return out
+}
+
+// icmpRe finds an ICMP type match in a raw rule: shown as just icmp or
+// icmpv6.
+var icmpRe = regexp.MustCompile(`\b(icmpv6|icmp) type\b`)
+
+// short sums up an exception for the overview: its services (ICMP types
+// as just icmp or icmpv6), with the addresses it is limited to, which
+// matter as much ("tcp/22 from trusted"); times and rates are left to
+// the details.
+func short(r *model.Rule, what string) string {
+	var parts []string
+	if len(r.Services) > 0 {
+		var svcs []string
+		seen := map[string]bool{}
+		for _, s := range r.Services {
+			if p, _, ok := strings.Cut(s, "/"); ok && (p == "icmp" || p == "icmpv6") {
+				s = p
+			}
+			if !seen[s] {
+				seen[s] = true
+				svcs = append(svcs, s)
+			}
+		}
+		parts = append(parts, strings.Join(svcs, ", "))
+	}
+	for _, c := range strings.Split(what, " · ") {
+		if strings.HasPrefix(c, "from ") || strings.HasPrefix(c, "to ") {
+			parts = append(parts, c)
+		}
+	}
+	if len(parts) == 0 {
+		first, _, _ := strings.Cut(what, " · ")
+		return first
+	}
+	return strings.Join(parts, " ")
+}
+
+// conditions says what limits a rule to part of a pair's traffic ("" for
+// none: it decides the pair).
+func conditions(st *model.State, r *model.Rule) string {
+	var parts []string
+	list := func(not bool, xs []string) string {
+		s := strings.Join(xs, ", ")
+		if not {
+			return "any but " + s
+		}
+		return s
+	}
+	if len(r.Sources) > 0 {
+		parts = append(parts, "from "+list(r.NotSources, r.Sources))
+	}
+	if len(r.Destinations) > 0 {
+		parts = append(parts, "to "+list(r.NotDests, r.Destinations))
+	}
+	if len(r.Services) > 0 {
+		parts = append(parts, strings.Join(r.Services, ", "))
+	}
+	if r.Family != "" {
+		parts = append(parts, r.Family+" only")
+	}
+	if len(r.CtStatus) > 0 {
+		parts = append(parts, "connections that were "+strings.Join(r.CtStatus, ", ")+"'d")
+	}
+	if len(r.Days) > 0 {
+		parts = append(parts, strings.Join(r.Days, ", "))
+	}
+	if r.HourFrom != "" {
+		parts = append(parts, r.HourFrom+"–"+r.HourTo)
+	}
+	if l := r.Limit; l != nil {
+		unit := ""
+		if l.Unit != "" && l.Unit != "packets" {
+			unit = " " + l.Unit
+		}
+		rate := fmt.Sprintf("%d%s/%s", l.Rate, unit, l.Per)
+		if l.Over {
+			parts = append(parts, "beyond "+rate)
+		} else {
+			parts = append(parts, "within "+rate)
+		}
+	}
+	if r.ConnLimit > 0 {
+		parts = append(parts, fmt.Sprintf("sources with over %d connections", r.ConnLimit))
+	}
+	if q := r.Quota; q != nil {
+		if q.Over {
+			parts = append(parts, "once its quota is used up")
+		} else {
+			parts = append(parts, "while its quota lasts")
+		}
+	}
+	return strings.Join(parts, " · ")
+}

internal/matrix/matrix_test.go

@@ -0,0 +1,199 @@
+package matrix
+
+import (
+	"fmt"
+	"net/netip"
+	"strings"
+	"testing"
+
+	"housecarl/internal/api"
+	"housecarl/internal/model"
+)
+
+func state() *model.State {
+	st := model.NewState()
+	st.Zones = []*model.Zone{
+		{ID: 1, Name: "guest", Interfaces: []string{"eth2"}},
+		{ID: 2, Name: "lan", Interfaces: []string{"eth1"}},
+		{ID: 3, Name: "wan", Interfaces: []string{"wan"}, External: true},
+	}
+	st.Sets = []*model.AddressSet{{ID: 4, Name: "admins", Entries: []string{"192.168.1.0/28"}}}
+	pos := 0.0
+	rule := func(r model.Rule) {
+		pos++
+		r.ID, r.Position = int64(10+pos), pos
+		st.Rules = append(st.Rules, &r)
+	}
+	rule(model.Rule{Chain: "input", Match: model.Match{Sources: []string{"admins"}, Services: []string{"ssh"}}, Action: "accept", Comment: "SSH"})
+	rule(model.Rule{Chain: "input", Match: model.Match{InZones: []string{"lan", "guest"}, Services: []string{"dns"}}, Action: "accept"})
+	rule(model.Rule{Chain: "forward", Match: model.Match{InZones: []string{"lan"}, OutZones: []string{"wan"}}, Action: "accept", Comment: "LAN out"})
+	rule(model.Rule{Chain: "forward", Match: model.Match{InZones: []string{"guest"}, OutZones: []string{"wan"}, Services: []string{"http", "https"}}, Action: "accept"})
+	rule(model.Rule{Chain: "forward", Match: model.Match{InZones: []string{"wan"}}, Action: "drop", Comment: "nothing in"})
+	rule(model.Rule{Chain: "forward", Match: model.Match{InZones: []string{"lan"}}, CtState: []string{"established"}, Action: "drop"}) // not for new
+	rule(model.Rule{Chain: "output", Match: model.Match{OutZones: []string{"wan"}}, Action: "drop", Disabled: true})
+	st.Forwards = []*model.PortForward{{ID: 30, Name: "web", Protocols: []string{"tcp"}, Ports: "443", To: "192.168.1.10"}}
+	st.Settings.Builtins.MSSClamp = true
+	return st
+}
+
+func find(m api.Matrix, from, to string) api.MatrixCell {
+	for _, c := range m.Cells {
+		if c.From == from && c.To == to {
+			return c
+		}
+	}
+	return api.MatrixCell{}
+}
+
+func TestMatrix(t *testing.T) {
+	m := Build(state())
+	if strings.Join(m.Zones, ",") != "guest,lan,wan" || len(m.Cells) != 3*3+3+3 {
+		t.Fatalf("zones %v, %d cells", m.Zones, len(m.Cells))
+	}
+	for _, c := range []struct {
+		from, to, verdict, decided string
+		exceptions                 int
+	}{
+		{"lan", "wan", "allowed", "rule forward #13 (LAN out)", 0},
+		{"guest", "wan", "blocked", "forward policy", 1},              // only web
+		{"wan", "lan", "blocked", "rule forward #15 (nothing in)", 2}, // antispoof, port forwards come first
+		{"lan", "guest", "blocked", "forward policy", 0},              // the established-only rule is no exception
+		{"lan", Firewall, "blocked", "input policy", 4},               // icmpv6, ping, ssh from admins, dns
+		{"wan", Firewall, "blocked", "input policy", 4},               // antispoof, icmpv6, dhcpv6, ssh from admins
+		{Firewall, "wan", "allowed", "output policy", 0},              // the drop is disabled
+	} {
+		got := find(m, c.from, c.to)
+		if got.Verdict != c.verdict || got.DecidedBy != c.decided || len(got.Exceptions) != c.exceptions {
+			t.Errorf("%s → %s: %s by %s, %d exceptions %+v; want %s by %s, %d", c.from, c.to, got.Verdict, got.DecidedBy, len(got.Exceptions), got.Exceptions, c.verdict, c.decided, c.exceptions)
+		}
+	}
+	g := find(m, "guest", "wan").Exceptions[0]
+	if g.Verdict != "allowed" || g.What != "http, https" || g.RuleID != 14 {
+		t.Errorf("the guest exception: %+v", g)
+	}
+	if e := find(m, "wan", "lan").Exceptions; e[0].Label != "built-in: antispoof" || e[1].Label != "port forwards" {
+		t.Errorf("wan → lan exceptions: %+v", e)
+	}
+	if _, ok := Cell(state(), Options{}, "lan", "nowhere"); ok {
+		t.Error("an unknown zone")
+	}
+}
+
+// Rules whose addresses all live behind another zone are left out of a
+// pair; routed (unknown) networks, and mixed lists, stay.
+func TestConnected(t *testing.T) {
+	st := state()
+	st.Sets = append(st.Sets, &model.AddressSet{ID: 5, Name: "guest_net", Entries: []string{"192.168.2.0/24"}},
+		&model.AddressSet{ID: 6, Name: "mixed", Entries: []string{"192.168.2.0/24", "10.20.30.0/24"}})
+	st.Rules = append(st.Rules,
+		&model.Rule{ID: 40, Chain: "forward", Position: 10, Match: model.Match{Sources: []string{"guest_net"}}, Action: "drop", Comment: "guests"},
+		&model.Rule{ID: 41, Chain: "forward", Position: 11, Match: model.Match{Destinations: []string{"192.168.2.5"}}, Action: "accept", Comment: "to a guest"},
+		&model.Rule{ID: 42, Chain: "forward", Position: 12, Match: model.Match{Sources: []string{"mixed"}}, Action: "drop", Comment: "mixed"})
+	o := Options{Connected: map[netip.Prefix]string{netip.MustParsePrefix("192.168.1.0/24"): "lan", netip.MustParsePrefix("192.168.2.0/24"): "guest"}}
+	ids := func(c api.MatrixCell) []int64 {
+		var out []int64
+		for _, e := range c.Exceptions {
+			out = append(out, e.RuleID)
+		}
+		return out
+	}
+	lanWan, _ := Cell(st, o, "lan", "wan")
+	if fmt.Sprint(ids(lanWan)) != "[]" {
+		t.Errorf("lan → wan: %v", ids(lanWan)) // decided by "LAN out" before the others anyway
+	}
+	lanGuest, _ := Cell(st, o, "lan", "guest")
+	if fmt.Sprint(ids(lanGuest)) != "[41 42]" {
+		t.Errorf("lan → guest: %v", ids(lanGuest)) // not "guests": their addresses are guest's
+	}
+	guestLan, _ := Cell(st, o, "guest", "lan")
+	if fmt.Sprint(ids(guestLan)) != "[40 42]" {
+		t.Errorf("guest → lan: %v", ids(guestLan)) // not "to a guest": it is in guest, not lan
+	}
+}
+
+// TestSummary: each pair at a glance, what its exceptions let through,
+// and the counters behind each line.
+func TestSummary(t *testing.T) {
+	st := state()
+	st.Settings.InputPolicy = model.PolicyDrop
+	m := Build(st)
+	for _, c := range []struct {
+		from, to, summary string
+		allowed, blocked  int
+		open              string
+	}{
+		{"lan", "wan", "allowed", 0, 0, ""},
+		{"guest", "wan", "partly", 1, 0, "http, https"},
+		{"wan", "lan", "partly", 1, 1, "1 port forward"},
+		{"lan", "guest", "blocked", 0, 0, ""},
+		{"lan", Firewall, "partly", 4, 0, "ICMPv6|ping|ssh from admins|dns"},
+	} {
+		got := find(m, c.from, c.to)
+		if got.Summary != c.summary || got.Allowed != c.allowed || got.Blocked != c.blocked || strings.Join(got.Open, "|") != c.open {
+			t.Errorf("%s → %s: %s +%d -%d %q; want %s +%d -%d %q", c.from, c.to, got.Summary, got.Allowed, got.Blocked, got.Open, c.summary, c.allowed, c.blocked, c.open)
+		}
+	}
+	if c := find(m, "lan", "lan"); !c.SameZone || !strings.Contains(c.Note, "out of eth1") {
+		t.Errorf("same zone: %+v", c)
+	}
+	if c := find(m, "lan", "wan"); c.SameZone || c.Note != "" {
+		t.Errorf("not the same zone: %+v", c)
+	}
+	wl := find(m, "wan", "lan")
+	if strings.Join(wl.Exceptions[0].Marks, ",") != "b-antispoof,b-antispoof6" || strings.Join(wl.Exceptions[1].Marks, ",") != "f30" || strings.Join(wl.Marks, ",") != "r15" {
+		t.Errorf("marks: %+v", wl)
+	}
+	icmp := &model.Rule{Match: model.Match{Services: []string{"icmp/echo-request", "icmp/echo-reply", "icmpv6/echo-request", "ssh"}}}
+	if s := short(icmp, "x"); s != "icmp, icmpv6, ssh" {
+		t.Errorf("short: %q", s)
+	}
+	ssh := &model.Rule{Match: model.Match{Sources: []string{"trusted"}, Services: []string{"ssh"}}, Days: []string{"mon"}}
+	if s := short(ssh, conditions(st, ssh)); s != "ssh from trusted" {
+		t.Errorf("short with a source: %q", s)
+	}
+	if s := short(&model.Rule{Days: []string{"mon"}}, "mon"); s != "mon" {
+		t.Errorf("short of a time: %q", s)
+	}
+	if c := find(m, "lan", Firewall); strings.Join(c.Marks, ",") != "p-input" {
+		t.Errorf("a drop policy's mark: %v", c.Marks)
+	}
+}
+
+// TestRaw: raw rules that are a match and a verdict are read; others stay
+// unknown.
+func TestRaw(t *testing.T) {
+	for _, c := range []struct {
+		raw, verdict, short string
+		ok, forNew          bool
+	}{
+		{"icmpv6 type { nd-router-solicit, nd-neighbor-advert } ip6 hoplimit 255 accept", "allowed", "icmpv6", true, true},
+		{`tcp dport 22 counter log prefix "ssh: " drop comment "x"`, "blocked", "tcp dport 22", true, true},
+		{"ip saddr 10.0.0.1 reject with icmpx type admin-prohibited", "blocked", "ip saddr 10.0.0.1", true, true},
+		{"ct state established,related accept", "allowed", "ct state established,related", true, false},
+		{"ct state { new, untracked } tcp dport 80 accept", "allowed", "ct state { new, untracked } tcp dport 80", true, true},
+		{"counter accept", "allowed", "", true, true},
+		{`iifname "wan" tcp dport 22 accept`, "", "", false, false},
+		{"tcp dport vmap { 22 : accept }", "", "", false, false},
+		{"meta mark set 1 accept", "", "", false, false},
+		{"tcp dport 22 counter", "", "", false, false},
+	} {
+		rr := readRaw(c.raw)
+		if rr.ok != c.ok || c.ok && (rr.verdict != c.verdict || rr.short != c.short || rr.forNew != c.forNew) {
+			t.Errorf("%q: %+v", c.raw, rr)
+		}
+	}
+	// In a pair: an exception that lets ND in, not "unknown".
+	st := state()
+	st.Rules = append(st.Rules, &model.Rule{ID: 90, Chain: "input", Position: 0.5, Raw: "icmpv6 type { nd-router-solicit } ip6 hoplimit 255 accept"})
+	st.Sort()
+	e := find(Build(st), "wan", Firewall).Exceptions
+	found := false
+	for _, x := range e {
+		if x.RuleID == 90 {
+			found = x.Verdict == "allowed" && x.Short == "icmpv6" && strings.HasPrefix(x.What, "raw: icmpv6 type")
+		}
+	}
+	if !found {
+		t.Errorf("the raw ND rule: %+v", e)
+	}
+}

internal/model/addr.go

@@ -0,0 +1,119 @@
+package model
+
+import (
+	"fmt"
+	"net/netip"
+	"strings"
+)
+
+// Address is a parsed address entry: one address, a prefix or a range.
+type Address struct {
+	Prefix     netip.Prefix // a single address is a /32 or /128
+	First, End netip.Addr   // a range (First valid only for ranges)
+}
+
+// IsV6 reports whether the entry is IPv6.
+func (a Address) IsV6() bool {
+	if a.First.IsValid() {
+		return a.First.Is6()
+	}
+	return a.Prefix.Addr().Is6()
+}
+
+// String writes the entry as nftables and Housecarl write it:
+// "10.0.0.5", "10.0.0.0/24", "10.0.0.1-10.0.0.9".
+func (a Address) String() string {
+	if a.First.IsValid() {
+		return a.First.String() + "-" + a.End.String()
+	}
+	if a.Prefix.IsSingleIP() {
+		return a.Prefix.Addr().String()
+	}
+	return a.Prefix.String()
+}
+
+// Contains reports whether the entry holds an address.
+func (a Address) Contains(x netip.Addr) bool {
+	x = x.Unmap()
+	if a.First.IsValid() {
+		return x.Is4() == a.First.Is4() && !x.Less(a.First) && !a.End.Less(x)
+	}
+	return a.Prefix.Contains(x)
+}
+
+// Overlaps reports whether the entry and a prefix share an address.
+func (a Address) Overlaps(p netip.Prefix) bool {
+	if !a.First.IsValid() {
+		return a.Prefix.Overlaps(p)
+	}
+	if a.First.Is4() != p.Addr().Is4() {
+		return false
+	}
+	// The prefix's first and last addresses against the range.
+	first := p.Masked().Addr()
+	last := first
+	b := last.AsSlice()
+	for i := p.Bits(); i < len(b)*8; i++ {
+		b[i/8] |= 0x80 >> (i % 8)
+	}
+	last, _ = netip.AddrFromSlice(b)
+	return !last.Less(a.First) && !a.End.Less(first)
+}
+
+// ParseAddress reads an address, a prefix (host bits must be zero) or a
+// range, IPv4 or IPv6.
+func ParseAddress(s string) (Address, error) {
+	s = strings.TrimSpace(s)
+	if a, b, ok := strings.Cut(s, "-"); ok {
+		first, err1 := netip.ParseAddr(strings.TrimSpace(a))
+		end, err2 := netip.ParseAddr(strings.TrimSpace(b))
+		if err1 != nil || err2 != nil || first.Is4() != end.Is4() {
+			return Address{}, fmt.Errorf("%q is not an address range (write it as 10.0.0.1-10.0.0.9)", s)
+		}
+		first, end = first.Unmap(), end.Unmap()
+		if end.Less(first) {
+			return Address{}, fmt.Errorf("the range %q ends before it starts", s)
+		}
+		if first == end {
+			return Address{Prefix: netip.PrefixFrom(first, first.BitLen())}, nil
+		}
+		return Address{First: first, End: end}, nil
+	}
+	if strings.Contains(s, "/") {
+		p, err := netip.ParsePrefix(s)
+		if err != nil {
+			return Address{}, fmt.Errorf("%q is not an address or prefix", s)
+		}
+		if p.Addr().Is4In6() {
+			// Unmapping would leave an IPv4 address with IPv6 bits
+			// (::ffff:0:0/96 as 0.0.0.0/96).
+			return Address{}, fmt.Errorf("%q is an IPv4-mapped prefix, which can't be used: write IPv4 prefixes as IPv4 (10.0.0.0/8)", s)
+		}
+		if p.Masked() != p {
+			return Address{}, fmt.Errorf("%q has host bits set; the prefix is %s", s, p.Masked())
+		}
+		return Address{Prefix: p}, nil
+	}
+	a, err := netip.ParseAddr(s)
+	if err != nil || a.Zone() != "" {
+		return Address{}, fmt.Errorf("%q is not an address, prefix, range or set name", s)
+	}
+	a = a.Unmap()
+	return Address{Prefix: netip.PrefixFrom(a, a.BitLen())}, nil
+}
+
+// IsLiteral reports whether a source/destination reference is an
+// address, prefix or range rather than a set name.
+func IsLiteral(ref string) bool {
+	_, err := ParseAddress(ref)
+	return err == nil
+}
+
+// NormalizeAddress rewrites an address entry in its canonical form.
+func NormalizeAddress(s string) (string, error) {
+	a, err := ParseAddress(s)
+	if err != nil {
+		return "", err
+	}
+	return a.String(), nil
+}

internal/model/addr_test.go

@@ -0,0 +1,28 @@
+package model
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseAddress(t *testing.T) {
+	for _, c := range []struct{ in, want, err string }{
+		{in: "10.0.0.0/8", want: "10.0.0.0/8"},
+		{in: "2001:db8::/32", want: "2001:db8::/32"},
+		{in: "::ffff:10.0.0.1", want: "10.0.0.1"},
+		{in: "10.0.0.1/8", err: "host bits"},
+		{in: "::ffff:0:0/96", err: "IPv4-mapped"},
+		{in: "::ffff:10.0.0.0/104", err: "IPv4-mapped"},
+	} {
+		a, err := ParseAddress(c.in)
+		if c.err != "" {
+			if err == nil || !strings.Contains(err.Error(), c.err) {
+				t.Errorf("%s: error %v, want one saying %q", c.in, err, c.err)
+			}
+			continue
+		}
+		if err != nil || a.String() != c.want {
+			t.Errorf("%s: %q, %v; want %q", c.in, a.String(), err, c.want)
+		}
+	}
+}

internal/model/auth.go

@@ -0,0 +1,134 @@
+package model
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// Grant gives access to one resource (see internal/auth).
+type Grant struct {
+	Resource string `json:"resource"` // config, sets, bans, log, audit, *
+	Access   string `json:"access"`   // read or write (write implies read)
+	// Sets limits a "sets" grant to these dynamic sets, and a "bans"
+	// grant to these ban lists (none: all).
+	Sets []string `json:"sets,omitempty"`
+}
+
+// User is a person or a service account.
+type User struct {
+	ID           int64      `json:"id"`
+	Username     string     `json:"username"`
+	PasswordHash string     `json:"-"`
+	HasPassword  bool       `json:"has_password"`
+	Admin        bool       `json:"admin"`
+	Grants       []Grant    `json:"grants"`
+	Disabled     bool       `json:"disabled"`
+	Comment      string     `json:"comment"`
+	CreatedAt    time.Time  `json:"created_at"`
+	UpdatedAt    time.Time  `json:"updated_at"`
+	LastLoginAt  *time.Time `json:"last_login_at,omitempty"`
+}
+
+// Token is an API token. The secret is shown once, when it is made; only
+// its hash is stored.
+type Token struct {
+	ID         int64  `json:"id"`
+	UserID     int64  `json:"user_id"`
+	Username   string `json:"username,omitempty"`
+	Name       string `json:"name"`
+	PublicID   string `json:"public_id"`
+	SecretHash string `json:"-"`
+	// Grants restrict the token to part of its user's permissions; nil
+	// means all of them.
+	Grants     []Grant    `json:"grants"`
+	ExpiresAt  *time.Time `json:"expires_at,omitempty"`
+	CreatedAt  time.Time  `json:"created_at"`
+	LastUsedAt *time.Time `json:"last_used_at,omitempty"`
+	LastUsedIP string     `json:"last_used_ip,omitempty"`
+	RevokedAt  *time.Time `json:"revoked_at,omitempty"`
+	// RotatedAt is when the token last got a new secret (nil: never).
+	RotatedAt *time.Time `json:"rotated_at,omitempty"`
+	// After a rotation with a grace period, the previous secret keeps
+	// working until PrevExpiresAt.
+	PrevSecretHash string     `json:"-"`
+	PrevExpiresAt  *time.Time `json:"previous_secret_valid_until,omitempty"`
+}
+
+// IssuedAt is when the token's current secret was made: its last
+// rotation, or its creation.
+func (t *Token) IssuedAt() time.Time {
+	if t.RotatedAt != nil {
+		return *t.RotatedAt
+	}
+	return t.CreatedAt
+}
+
+// Session is a web UI login.
+type Session struct {
+	IDHash     string    `json:"-"`
+	UserID     int64     `json:"user_id"`
+	CSRFToken  string    `json:"-"`
+	CreatedAt  time.Time `json:"created_at"`
+	ExpiresAt  time.Time `json:"expires_at"`
+	LastSeenAt time.Time `json:"last_seen_at"`
+	IP         string    `json:"ip"`
+	UserAgent  string    `json:"user_agent"`
+}
+
+// AuditEvent records an action: logins, users and tokens, changes,
+// dynamic set entries.
+type AuditEvent struct {
+	ID       int64     `json:"id"`
+	Time     time.Time `json:"time"`
+	UserID   int64     `json:"user_id,omitempty"`
+	Username string    `json:"username,omitempty"`
+	Auth     string    `json:"auth,omitempty"` // session, token, socket
+	TokenID  int64     `json:"token_id,omitempty"`
+	IP       string    `json:"ip,omitempty"`
+	Action   string    `json:"action"`
+	Object   string    `json:"object,omitempty"`
+	Result   string    `json:"result"` // success, failure, denied
+	Detail   string    `json:"detail,omitempty"`
+}
+
+// Change is one object's change: Before is nil for a create, After is nil
+// for a delete. Settings have ID 0.
+type Change struct {
+	Kind   string          `json:"kind"`
+	ID     int64           `json:"id"`
+	Label  string          `json:"label"`
+	Before json.RawMessage `json:"before,omitempty"`
+	After  json.RawMessage `json:"after,omitempty"`
+}
+
+// Action describes a change in a word.
+func (c Change) Action() string {
+	switch {
+	case c.Before == nil:
+		return "create"
+	case c.After == nil:
+		return "delete"
+	}
+	return "update"
+}
+
+// Inverse is the change that undoes c.
+func (c Change) Inverse() Change {
+	c.Before, c.After = c.After, c.Before
+	return c
+}
+
+// HistoryEntry is a saved configuration change.
+type HistoryEntry struct {
+	ID          int64     `json:"id"`
+	Time        time.Time `json:"time"`
+	UserID      int64     `json:"user_id,omitempty"`
+	Username    string    `json:"username"`
+	Via         string    `json:"via"` // web, cli, api
+	Description string    `json:"description"`
+	Changes     []Change  `json:"changes"`
+	// RevertOf is the entry this one undid; RevertedBy the entry that
+	// undid this one.
+	RevertOf   int64 `json:"revert_of,omitempty"`
+	RevertedBy int64 `json:"reverted_by,omitempty"`
+}

internal/model/bans.go

@@ -0,0 +1,53 @@
+package model
+
+import (
+	"net/netip"
+	"time"
+)
+
+// Ban is an address a ban list keeps out (docs/bans.md): of every port,
+// or of one protocol and port. Bans are kept apart from the
+// configuration, like a dynamic set's entries: not in the history, and
+// not on probation.
+type Ban struct {
+	ID     int64  `json:"id"`
+	ListID int64  `json:"list_id"`
+	List   string `json:"list"`
+	// Address is what is banned: an address, or an IPv6 prefix of the
+	// list's grouping ("2001:db8:1:2::/64").
+	Address  string `json:"address"`
+	Protocol string `json:"protocol,omitempty"` // tcp or udp; "" every port
+	Port     int    `json:"port,omitempty"`
+	// Expires is when the ban ends (nil: it doesn't); Timeout the
+	// seconds it has left then.
+	Expires *time.Time `json:"expires,omitempty"`
+	Timeout int        `json:"timeout,omitempty"`
+	Created time.Time  `json:"created"`
+	// Who banned it (the user or token's name), Source the program's own
+	// name for why ("guard/ssh", "fail2ban/sshd"), Reason free text (a
+	// log line).
+	Who    string `json:"who"`
+	Source string `json:"source,omitempty"`
+	Reason string `json:"reason,omitempty"`
+}
+
+// Prefix is the banned address as a prefix.
+func (b *Ban) Prefix() (netip.Prefix, error) {
+	if p, err := netip.ParsePrefix(b.Address); err == nil {
+		return p, nil
+	}
+	a, err := netip.ParseAddr(b.Address)
+	if err != nil {
+		return netip.Prefix{}, err
+	}
+	return netip.PrefixFrom(a, a.BitLen()), nil
+}
+
+// BanAddress writes a banned prefix: an address alone when it is one
+// address, else the prefix.
+func BanAddress(p netip.Prefix) string {
+	if p.IsSingleIP() {
+		return p.Addr().String()
+	}
+	return p.String()
+}

internal/model/changeset.go

@@ -0,0 +1,62 @@
+package model
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// Changeset statuses.
+const (
+	ChangesetOpen      = "open"
+	ChangesetApplied   = "applied"
+	ChangesetDiscarded = "discarded"
+)
+
+// Changeset is a list of changes, staged to be reviewed and applied
+// together (or discarded). Applied, it is one history entry.
+type Changeset struct {
+	ID          int64     `json:"id"`
+	UserID      int64     `json:"user_id,omitempty"`
+	Username    string    `json:"username"`
+	Via         string    `json:"via"`
+	Description string    `json:"description"`
+	Status      string    `json:"status"`
+	Ops         []Op      `json:"ops"`
+	CreatedAt   time.Time `json:"created_at"`
+	UpdatedAt   time.Time `json:"updated_at"`
+	// HistoryID is the history entry an applied changeset became.
+	HistoryID int64 `json:"history_id,omitempty"`
+}
+
+// Operations.
+const (
+	OpCreate   = "create"
+	OpUpdate   = "update"
+	OpDelete   = "delete"
+	OpMove     = "move"
+	OpSettings = "settings"
+	// OpRevert undoes a history entry: Data is the entry (its changes),
+	// ID its number.
+	OpRevert = "revert"
+)
+
+// Op is one change, as the API received it, so it can be stored and
+// replayed on whatever the configuration is when it is applied.
+type Op struct {
+	Seq  int    `json:"seq"`
+	Op   string `json:"op"` // create, update, delete, move, settings, revert
+	Kind string `json:"kind,omitempty"`
+	// Ref names the object (ID, or name) for update, delete and move.
+	Ref string `json:"ref,omitempty"`
+	// ID is the object a create makes, given when it is staged so later
+	// operations can name it.
+	ID int64 `json:"id,omitempty"`
+	// Data is the object (create), the fields to change (update,
+	// settings) or where to move it (move).
+	Data json.RawMessage `json:"data,omitempty"`
+	// Place is where a create puts the object in its order (a move's
+	// data); none: at the end.
+	Place   json.RawMessage `json:"place,omitempty"`
+	Summary string          `json:"summary"`
+	AddedAt time.Time       `json:"added_at"`
+}

internal/model/objects.go

@@ -0,0 +1,535 @@
+package model
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Chains rules are placed in.
+const (
+	// ChainEarly sees packets before connection tracking (prerouting,
+	// priority raw): cheap drops, antispoofing.
+	ChainEarly = "early"
+	// ChainInput is traffic to the firewall itself.
+	ChainInput = "input"
+	// ChainForward is traffic through the firewall.
+	ChainForward = "forward"
+	// ChainOutput is traffic from the firewall itself.
+	ChainOutput = "output"
+)
+
+// Chains lists the rule chains in display order.
+var Chains = []string{ChainEarly, ChainInput, ChainForward, ChainOutput}
+
+// ChainOrder is a chain's place in Chains.
+func ChainOrder(c string) int {
+	for i, x := range Chains {
+		if x == c {
+			return i
+		}
+	}
+	return len(Chains)
+}
+
+// Rule actions.
+const (
+	ActionAccept = "accept"
+	ActionDrop   = "drop"
+	ActionReject = "reject"
+	// ActionLog only logs (and counts); the packet goes on to the next
+	// rule.
+	ActionLog = "log"
+)
+
+// Actions lists the rule actions.
+var Actions = []string{ActionAccept, ActionDrop, ActionReject, ActionLog}
+
+// Families a rule can be limited to; "" is both.
+const (
+	FamilyIPv4 = "ipv4"
+	FamilyIPv6 = "ipv6"
+)
+
+// Zone is a group of interfaces (wildcards such as "incus*" allowed).
+// External zones face the internet: built-ins such as antispoofing and
+// MSS clamping apply to them.
+type Zone struct {
+	ID         int64    `json:"id"`
+	Name       string   `json:"name"`
+	Interfaces []string `json:"interfaces"`
+	External   bool     `json:"external,omitempty"`
+	Comment    string   `json:"comment,omitempty"`
+}
+
+func (z *Zone) GetID() int64  { return z.ID }
+func (z *Zone) Label() string { return z.Name }
+
+// AddressSet is a named list of IPv4 and IPv6 addresses, prefixes and
+// ranges. A dynamic set also takes entries at run time, each with a
+// timeout, through its own API (a captive portal's signed-in clients).
+type AddressSet struct {
+	ID      int64    `json:"id"`
+	Name    string   `json:"name"`
+	Entries []string `json:"entries"`
+	Dynamic bool     `json:"dynamic,omitempty"`
+	// Timeout is a dynamic set's default entry lifetime in seconds (0:
+	// entries stay until removed).
+	Timeout int `json:"timeout,omitempty"`
+	// Jarl makes the set's entries the addresses of names in Jarl, which
+	// housecarld reads and keeps up to date (docs/jarl.md).
+	Jarl    *JarlSource `json:"jarl,omitempty"`
+	Comment string      `json:"comment,omitempty"`
+}
+
+func (s *AddressSet) GetID() int64 { return s.ID }
+
+// Runtime reports whether the set's entries come at run time (a dynamic
+// set's, a Jarl set's) rather than from the configuration.
+func (s *AddressSet) Runtime() bool { return s.Dynamic || s.Jarl != nil }
+
+// JarlSource is where a set's addresses come from in Jarl: the A and AAAA
+// records of names (CNAMEs followed), each in a view, kept only where
+// Within allows.
+type JarlSource struct {
+	Names []JarlName `json:"names"`
+	// Within are zone names and addresses (prefixes, ranges): an address
+	// is used only if it lies in a zone's connected networks or in one of
+	// the addresses.
+	Within []string `json:"within"`
+}
+
+// JarlName is a name in Jarl, in a view ("": the only view that has it).
+type JarlName struct {
+	Name string `json:"name"` // "nas.example.com", without the final dot
+	View string `json:"view,omitempty"`
+}
+
+// String is "name" or "name@view".
+func (n JarlName) String() string {
+	if n.View == "" {
+		return n.Name
+	}
+	return n.Name + "@" + n.View
+}
+
+// ParseJarlName reads "name" or "name@view".
+func ParseJarlName(s string) JarlName {
+	name, view, _ := strings.Cut(strings.TrimSpace(s), "@")
+	return JarlName{Name: strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), "."), View: strings.TrimSpace(view)}
+}
+func (s *AddressSet) Label() string { return s.Name }
+
+// Service is a named list of ports ("tcp/22", "udp/60000-61000") and ICMP
+// types ("icmp/echo-request", "icmpv6/packet-too-big"). Rules name
+// services, or give such entries directly.
+type Service struct {
+	ID      int64    `json:"id"`
+	Name    string   `json:"name"`
+	Entries []string `json:"entries"`
+	Comment string   `json:"comment,omitempty"`
+	// Builtin marks the catalog's services (never stored).
+	Builtin bool `json:"builtin,omitempty"`
+}
+
+func (s *Service) GetID() int64  { return s.ID }
+func (s *Service) Label() string { return s.Name }
+
+// Match is what a rule, port forward, source NAT or redirect looks at.
+// Lists are ORed; the fields together are ANDed.
+type Match struct {
+	// InZones and OutZones are zone names; with NotIn/NotOut, every zone
+	// but those.
+	InZones  []string `json:"in_zones,omitempty"`
+	NotIn    bool     `json:"not_in,omitempty"`
+	OutZones []string `json:"out_zones,omitempty"`
+	NotOut   bool     `json:"not_out,omitempty"`
+	// Sources and Destinations are set names or literal addresses,
+	// prefixes and ranges, IPv4 or IPv6.
+	Sources      []string `json:"sources,omitempty"`
+	NotSources   bool     `json:"not_sources,omitempty"`
+	Destinations []string `json:"destinations,omitempty"`
+	NotDests     bool     `json:"not_destinations,omitempty"`
+	// Services are service names or entries ("tcp/22", "icmp/echo-request").
+	Services []string `json:"services,omitempty"`
+	// Family limits the rule to ipv4 or ipv6 ("": what its addresses
+	// and services allow).
+	Family string `json:"family,omitempty"`
+}
+
+// Rule is one line of a chain, checked in order.
+type Rule struct {
+	ID       int64   `json:"id"`
+	Chain    string  `json:"chain"`
+	Position float64 `json:"position"`
+	Match
+	// CtState: new, established, related, invalid, untracked.
+	CtState []string `json:"ct_state,omitempty"`
+	// CtStatus: dnat, snat (a connection that was NATed).
+	CtStatus []string `json:"ct_status,omitempty"`
+	// Days ("Monday", …) and hours ("08:00"-"17:30") the rule applies.
+	Days     []string `json:"days,omitempty"`
+	HourFrom string   `json:"hour_from,omitempty"`
+	HourTo   string   `json:"hour_to,omitempty"`
+	// Limit, if set, makes the rule match only packets within (or with
+	// LimitOver, beyond) a rate.
+	Limit *Limit `json:"limit,omitempty"`
+	// ConnLimit matches when a source already has more than this many
+	// connections (per source address, grouped as GroupV4/GroupV6 say).
+	ConnLimit int `json:"conn_limit,omitempty"`
+	// Quota, if set, is an allowance of bytes: the rule matches once it
+	// is used up (Over), or while it lasts.
+	Quota *Quota `json:"quota,omitempty"`
+	// GroupV4 and GroupV6 group addresses for the per-address limits,
+	// connection limit and quota: by prefix length (0: the default, each
+	// IPv4 address alone, IPv6 by /64, since one device has several).
+	GroupV4 int `json:"group_v4,omitempty"`
+	GroupV6 int `json:"group_v6,omitempty"`
+	// EveryPacket checks the rule against every packet, before established
+	// connections are accepted, so a drop or reject also stops connections
+	// already open, not just new ones.
+	EveryPacket bool   `json:"every_packet,omitempty"`
+	Action      string `json:"action"`
+	// RejectWith: "" (ICMP port/admin unreachable as fits) or tcp-reset.
+	RejectWith string `json:"reject_with,omitempty"`
+	Log        bool   `json:"log,omitempty"`
+	LogPrefix  string `json:"log_prefix,omitempty"`
+	// Raw, when set, is an nftables rule written by hand, used instead of
+	// the fields above (the escape hatch for what Housecarl can't say).
+	Raw      string `json:"raw,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+func (r *Rule) GetID() int64 { return r.ID }
+func (r *Rule) Label() string {
+	if r.Comment != "" {
+		return fmt.Sprintf("%s #%d (%s)", r.Chain, r.ID, r.Comment)
+	}
+	return fmt.Sprintf("%s #%d", r.Chain, r.ID)
+}
+
+// Limit is a rate: Rate packets (or bytes, with Unit) per Per (second,
+// minute, hour, day), with a burst.
+type Limit struct {
+	Rate int    `json:"rate"`
+	Per  string `json:"per"`
+	// Burst is in the rate's unit.
+	Burst int `json:"burst,omitempty"`
+	// Unit: "" for packets, or bytes, kbytes or mbytes for a byte rate
+	// (bandwidth).
+	Unit string `json:"unit,omitempty"`
+	// Over matches packets beyond the rate instead of within it (to drop
+	// floods).
+	Over bool `json:"over,omitempty"`
+	// PerSource counts each source address on its own (a meter), instead
+	// of all traffic the rule sees together; PerDestination each
+	// destination.
+	PerSource      bool `json:"per_source,omitempty"`
+	PerDestination bool `json:"per_destination,omitempty"`
+}
+
+// Byte units, as nftables writes them.
+var ByteUnits = map[string]int64{"bytes": 1, "kbytes": 1 << 10, "mbytes": 1 << 20, "gbytes": 1 << 30}
+
+// Quota is an allowance of bytes. nftables counts it; Housecarl keeps the
+// count across changes and restarts, and resets it each period.
+type Quota struct {
+	Bytes int64 `json:"bytes"`
+	// Over matches once the allowance is used up (to drop); otherwise the
+	// rule matches while it lasts (to accept).
+	Over bool `json:"over,omitempty"`
+	// PerSource gives each source address (grouped as the rule's
+	// GroupV4/GroupV6 say) its own allowance; PerDestination each
+	// destination.
+	PerSource      bool `json:"per_source,omitempty"`
+	PerDestination bool `json:"per_destination,omitempty"`
+	// Reset: "" (only by hand), daily, weekly (on Mondays) or monthly (on
+	// Day, 1-31: a short month's last day when it has no such day), at
+	// midnight.
+	Reset string `json:"reset,omitempty"`
+	Day   int    `json:"day,omitempty"`
+}
+
+// Quota resets.
+var QuotaResets = []string{"daily", "weekly", "monthly"}
+
+// PerAddress reports whether a quota (or limit) counts each address on its
+// own, and which: "saddr", "daddr" or "".
+func (q *Quota) PerAddress() string {
+	switch {
+	case q == nil:
+		return ""
+	case q.PerSource:
+		return "saddr"
+	case q.PerDestination:
+		return "daddr"
+	}
+	return ""
+}
+
+// PerAddress: as Quota's.
+func (l *Limit) PerAddress() string {
+	switch {
+	case l == nil:
+		return ""
+	case l.PerSource:
+		return "saddr"
+	case l.PerDestination:
+		return "daddr"
+	}
+	return ""
+}
+
+// CountsTraffic reports whether a rule measures traffic (a quota or a byte
+// rate): it must see every packet of a connection, so it is checked before
+// the built-in that accepts established connections.
+func (r *Rule) CountsTraffic() bool {
+	if r.Raw != "" {
+		return false
+	}
+	if r.Quota != nil {
+		return true
+	}
+	if r.Limit == nil {
+		return false
+	}
+	_, bytes := ByteUnits[r.Limit.Unit]
+	return bytes
+}
+
+// ChecksEveryPacket reports whether a rule is checked before the built-in
+// that accepts established connections: it counts traffic, or asks to.
+func (r *Rule) ChecksEveryPacket() bool {
+	return r.Raw == "" && (r.EveryPacket || r.CountsTraffic())
+}
+
+// Groups returns a rule's address grouping, defaults applied.
+func (r *Rule) Groups() (v4, v6 int) {
+	v4, v6 = r.GroupV4, r.GroupV6
+	if v4 == 0 {
+		v4 = 32
+	}
+	if v6 == 0 {
+		v6 = 64
+	}
+	return v4, v6
+}
+
+// What becomes of what a rule's quota used when the rule changes
+// (QuotaCarry).
+const (
+	QuotaKept      = ""          // counted as before: carried as it is
+	QuotaSummed    = "summed"    // per address to a total: the addresses' sum
+	QuotaRegrouped = "regrouped" // per address in wider groups: added up per group
+	QuotaRestarts  = "restarts"  // can't be carried: starts again
+)
+
+// QuotaCarry says what becomes of what old's quota used when the rule
+// becomes new (both the same rule, with quotas), and for QuotaRestarts
+// why. The amount, over and the resets don't matter: only how it counts.
+func QuotaCarry(old, new *Rule) (carry, why string) {
+	if old == nil || new == nil || old.Quota == nil || new.Quota == nil {
+		return QuotaKept, ""
+	}
+	was, is := old.Quota.PerAddress(), new.Quota.PerAddress()
+	words := map[string]string{"saddr": "per source", "daddr": "per destination", "": "in total"}
+	switch {
+	case was == "" && is == "":
+		return QuotaKept, ""
+	case is == "":
+		return QuotaSummed, ""
+	case was == "":
+		return QuotaRestarts, "a total can't be split by address, so each address starts at nothing"
+	case was != is:
+		return QuotaRestarts, "what was counted " + words[was] + " can't be counted " + words[is]
+	}
+	o4, o6 := old.Groups()
+	n4, n6 := new.Groups()
+	switch {
+	case o4 == n4 && o6 == n6:
+		return QuotaKept, ""
+	case n4 > o4 || n6 > o6:
+		return QuotaRestarts, "addresses are grouped more finely now, and a group's use can't be split"
+	}
+	return QuotaRegrouped, ""
+}
+
+// PortForward publishes a service on an inside host: incoming connections
+// to a public address and port are sent on (DNAT) to To[:ToPort], let
+// through the forward chain, and, with Hairpin, also work from inside.
+// To6 also opens the port to an IPv6 host (a pinhole; no translation).
+type PortForward struct {
+	ID       int64   `json:"id"`
+	Position float64 `json:"position"`
+	Name     string  `json:"name,omitempty"`
+	// Protocols: tcp, udp (default tcp).
+	Protocols []string `json:"protocols"`
+	// Ports is the public port or range ("8096", "7777-7780").
+	Ports string `json:"ports"`
+	// PublicAddresses are the IPv4 addresses (or set names) to match;
+	// none means any address on the external zones' interfaces (and then
+	// no hairpin).
+	PublicAddresses []string `json:"public_addresses,omitempty"`
+	// Sources limits who may connect (set names or addresses).
+	Sources []string `json:"sources,omitempty"`
+	To      string   `json:"to,omitempty"`
+	// ToPort is the inside port ("" keeps the public one; for a range, the
+	// first port of an equally long range).
+	ToPort string `json:"to_port,omitempty"`
+	To6    string `json:"to6,omitempty"`
+	// Hairpin lets inside clients use the public address: their
+	// connections are masqueraded so replies come back through the
+	// firewall. HairpinSources limits that to some sources (none: all
+	// inside traffic).
+	Hairpin        bool     `json:"hairpin,omitempty"`
+	HairpinSources []string `json:"hairpin_sources,omitempty"`
+	Comment        string   `json:"comment,omitempty"`
+	Disabled       bool     `json:"disabled,omitempty"`
+}
+
+func (f *PortForward) GetID() int64 { return f.ID }
+func (f *PortForward) Label() string {
+	if f.Name != "" {
+		return f.Name
+	}
+	return fmt.Sprintf("port forward #%d (%s)", f.ID, f.Ports)
+}
+
+// SourceNAT rewrites the source address of outgoing traffic: masquerade
+// (To empty: the outgoing interface's address) or SNAT to To.
+type SourceNAT struct {
+	ID       int64   `json:"id"`
+	Position float64 `json:"position"`
+	Match
+	To       string `json:"to,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+func (s *SourceNAT) GetID() int64 { return s.ID }
+func (s *SourceNAT) Label() string {
+	if s.Comment != "" {
+		return fmt.Sprintf("source NAT #%d (%s)", s.ID, s.Comment)
+	}
+	return fmt.Sprintf("source NAT #%d", s.ID)
+}
+
+// Redirect sends matching traffic to a port on the firewall itself (to
+// make every client use its DNS server, say).
+type Redirect struct {
+	ID       int64   `json:"id"`
+	Position float64 `json:"position"`
+	Match
+	// ToPort is the local port (the services' first port when empty).
+	ToPort   string `json:"to_port,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+func (r *Redirect) GetID() int64 { return r.ID }
+func (r *Redirect) Label() string {
+	if r.Comment != "" {
+		return fmt.Sprintf("redirect #%d (%s)", r.ID, r.Comment)
+	}
+	return fmt.Sprintf("redirect #%d", r.ID)
+}
+
+// Ban list scopes: what a list's bans keep the banned addresses from.
+const (
+	// BanScopeFirewall: the firewall itself (the input chain).
+	BanScopeFirewall = "firewall"
+	// BanScopeHosts: the list's hosts, behind the firewall (the forward
+	// chain, where port forwards' addresses are already translated).
+	BanScopeHosts = "hosts"
+	// BanScopeEverything: the firewall and everything behind it.
+	BanScopeEverything = "everything"
+)
+
+// BanScopes lists the scopes.
+var BanScopes = []string{BanScopeFirewall, BanScopeHosts, BanScopeEverything}
+
+// Ban lists' defaults.
+const (
+	DefaultBanTime    = 3600
+	DefaultBanMaxTime = 7 * 24 * 3600
+	DefaultBanEntries = 10000
+	DefaultBanGroupV6 = 64
+)
+
+// BanList is where a program such as Guard, or fail2ban, bans addresses
+// (docs/bans.md). The program names only the address, and perhaps a
+// protocol and ports; the list says what the address is kept from. The
+// bans themselves are kept apart from the configuration, like a dynamic
+// set's entries.
+type BanList struct {
+	ID    int64  `json:"id"`
+	Name  string `json:"name"`
+	Scope string `json:"scope"`
+	// Hosts are set names or addresses: what scope hosts protects.
+	Hosts []string `json:"hosts,omitempty"`
+	// Never are set names or addresses the list never bans.
+	Never []string `json:"never,omitempty"`
+	// DefaultTime is a ban's length in seconds when none is asked for
+	// (0: an hour); MaxTime the longest (0: a week; -1: no limit, so
+	// bans may be permanent).
+	DefaultTime int `json:"default_time,omitempty"`
+	MaxTime     int `json:"max_time,omitempty"`
+	// MaxEntries limits the bans the list holds (0: 10000).
+	MaxEntries int `json:"max_entries,omitempty"`
+	// GroupV6 is the IPv6 prefix length banned for an address (0: /64,
+	// what one customer usually gets); IPv4 addresses are banned alone.
+	GroupV6  int    `json:"group_v6,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+	Disabled bool   `json:"disabled,omitempty"`
+}
+
+func (b *BanList) GetID() int64  { return b.ID }
+func (b *BanList) Label() string { return b.Name }
+
+// BanTime is the default ban length in seconds.
+func (b *BanList) BanTime() int {
+	if b.DefaultTime > 0 {
+		return b.DefaultTime
+	}
+	return DefaultBanTime
+}
+
+// BanMaxTime is the longest ban in seconds, 0 for no limit.
+func (b *BanList) BanMaxTime() int {
+	switch {
+	case b.MaxTime < 0:
+		return 0
+	case b.MaxTime > 0:
+		return b.MaxTime
+	}
+	return DefaultBanMaxTime
+}
+
+// Entries is the most bans the list holds.
+func (b *BanList) Entries() int {
+	if b.MaxEntries > 0 {
+		return b.MaxEntries
+	}
+	return DefaultBanEntries
+}
+
+// V6Group is the IPv6 prefix length an address is banned with.
+func (b *BanList) V6Group() int {
+	if b.GroupV6 > 0 {
+		return b.GroupV6
+	}
+	return DefaultBanGroupV6
+}
+
+// Covers reports whether the list's bans apply to a chain.
+func (b *BanList) Covers(chain string) bool {
+	switch b.Scope {
+	case BanScopeFirewall:
+		return chain == ChainInput
+	case BanScopeHosts:
+		return chain == ChainForward
+	case BanScopeEverything:
+		return chain == ChainInput || chain == ChainForward
+	}
+	return false
+}

internal/model/quota_test.go

@@ -0,0 +1,26 @@
+package model
+
+import "testing"
+
+func TestQuotaCarry(t *testing.T) {
+	rule := func(per string, v4, v6 int) *Rule {
+		return &Rule{GroupV4: v4, GroupV6: v6, Quota: &Quota{Bytes: 1, PerSource: per == "src", PerDestination: per == "dst"}}
+	}
+	for _, c := range []struct {
+		old, new *Rule
+		want     string
+	}{
+		{rule("", 0, 0), rule("", 0, 0), QuotaKept},
+		{rule("src", 0, 0), rule("src", 32, 64), QuotaKept}, // the defaults, written out
+		{rule("src", 0, 0), rule("", 0, 0), QuotaSummed},
+		{rule("", 0, 0), rule("src", 0, 0), QuotaRestarts},
+		{rule("src", 0, 0), rule("dst", 0, 0), QuotaRestarts},
+		{rule("src", 0, 0), rule("src", 24, 56), QuotaRegrouped},
+		{rule("src", 0, 0), rule("src", 24, 128), QuotaRestarts},
+		{nil, rule("src", 0, 0), QuotaKept},
+	} {
+		if got, _ := QuotaCarry(c.old, c.new); got != c.want {
+			t.Errorf("%+v -> %+v: %q, want %q", c.old, c.new, got, c.want)
+		}
+	}
+}

internal/model/resolve.go

@@ -0,0 +1,96 @@
+package model
+
+// FamilyRefs are a reference list's part for one address family: names of
+// sets that have entries of the family (dynamic and Jarl sets always
+// do), and
+// literal entries.
+type FamilyRefs struct {
+	Sets     []string
+	Literals []string
+}
+
+// Empty reports whether nothing of the family was referenced.
+func (f FamilyRefs) Empty() bool { return len(f.Sets) == 0 && len(f.Literals) == 0 }
+
+// SplitRefs splits source or destination references by family. Unknown
+// names are left out (validation reports them).
+func (st *State) SplitRefs(refs []string) (v4, v6 FamilyRefs) {
+	for _, r := range refs {
+		if a, err := ParseAddress(r); err == nil {
+			if a.IsV6() {
+				v6.Literals = append(v6.Literals, a.String())
+			} else {
+				v4.Literals = append(v4.Literals, a.String())
+			}
+			continue
+		}
+		s := st.Set(r)
+		if s == nil {
+			continue
+		}
+		has4, has6 := s.Families()
+		if has4 {
+			v4.Sets = append(v4.Sets, s.Name)
+		}
+		if has6 {
+			v6.Sets = append(v6.Sets, s.Name)
+		}
+	}
+	return v4, v6
+}
+
+// Families reports which families a set holds: a dynamic or Jarl set may
+// get either at run time.
+func (s *AddressSet) Families() (v4, v6 bool) {
+	if s.Runtime() {
+		return true, true
+	}
+	for _, e := range s.Entries {
+		if a, err := ParseAddress(e); err == nil {
+			if a.IsV6() {
+				v6 = true
+			} else {
+				v4 = true
+			}
+		}
+	}
+	return v4, v6
+}
+
+// FamilyEntries returns a set's static entries of one family.
+func (s *AddressSet) FamilyEntries(v6 bool) []string {
+	var out []string
+	for _, e := range s.Entries {
+		if a, err := ParseAddress(e); err == nil && a.IsV6() == v6 {
+			out = append(out, a.String())
+		}
+	}
+	return out
+}
+
+// ServiceEntries resolves service references (names and entries) to
+// entries, without duplicates. Unknown names are left out.
+func (st *State) ServiceEntries(refs []string) []ServiceEntry {
+	var out []ServiceEntry
+	seen := map[string]bool{}
+	add := func(s string) {
+		e, err := ParseServiceEntry(s)
+		if err != nil || seen[e.String()] {
+			return
+		}
+		seen[e.String()] = true
+		out = append(out, e)
+	}
+	for _, r := range refs {
+		if IsServiceEntry(r) {
+			add(r)
+			continue
+		}
+		if svc := st.Service(r); svc != nil {
+			for _, e := range svc.Entries {
+				add(e)
+			}
+		}
+	}
+	return out
+}

internal/model/services.go

@@ -0,0 +1,146 @@
+package model
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// ServiceEntry is a parsed service entry: ports of a transport protocol
+// ("tcp/22", "udp/60000-61000") or an ICMP type ("icmp/echo-request",
+// "icmpv6/packet-too-big").
+type ServiceEntry struct {
+	Proto string // tcp, udp, sctp, icmp, icmpv6
+	// Ports: "22" or "1000-2000" (transport protocols).
+	Ports string
+	// Type: the ICMP type's name (icmp, icmpv6).
+	Type string
+}
+
+func (e ServiceEntry) String() string {
+	if e.Type != "" {
+		return e.Proto + "/" + e.Type
+	}
+	return e.Proto + "/" + e.Ports
+}
+
+// IsICMP reports whether the entry is an ICMP or ICMPv6 type.
+func (e ServiceEntry) IsICMP() bool { return e.Proto == "icmp" || e.Proto == "icmpv6" }
+
+// ICMP types nftables knows, by protocol.
+var icmpTypes = map[string][]string{
+	"icmp": {"echo-reply", "destination-unreachable", "source-quench", "redirect", "echo-request",
+		"router-advertisement", "router-solicitation", "time-exceeded", "parameter-problem",
+		"timestamp-request", "timestamp-reply", "info-request", "info-reply", "address-mask-request",
+		"address-mask-reply"},
+	"icmpv6": {"destination-unreachable", "packet-too-big", "time-exceeded", "parameter-problem",
+		"echo-request", "echo-reply", "mld-listener-query", "mld-listener-report", "mld-listener-done",
+		"mld-listener-reduction", "nd-router-solicit", "nd-router-advert", "nd-neighbor-solicit",
+		"nd-neighbor-advert", "nd-redirect", "router-renumbering", "ind-neighbor-solicit",
+		"ind-neighbor-advert", "mld2-listener-report"},
+}
+
+// ParseServiceEntry reads "tcp/22", "udp/1000-2000", "tcp/ssh"-style
+// entries are not accepted (use the service's name instead).
+func ParseServiceEntry(s string) (ServiceEntry, error) {
+	s = strings.ToLower(strings.TrimSpace(s))
+	proto, rest, ok := strings.Cut(s, "/")
+	if !ok {
+		return ServiceEntry{}, fmt.Errorf("%q is not a service entry (tcp/22, udp/1000-2000, icmp/echo-request) or a service name", s)
+	}
+	switch proto {
+	case "tcp", "udp", "sctp":
+		ports, err := NormalizePorts(rest)
+		if err != nil {
+			return ServiceEntry{}, fmt.Errorf("%q: %v", s, err)
+		}
+		return ServiceEntry{Proto: proto, Ports: ports}, nil
+	case "icmp", "icmpv6":
+		for _, t := range icmpTypes[proto] {
+			if t == rest {
+				return ServiceEntry{Proto: proto, Type: rest}, nil
+			}
+		}
+		return ServiceEntry{}, fmt.Errorf("%q: unknown %s type (%s)", s, proto, strings.Join(icmpTypes[proto], ", "))
+	}
+	return ServiceEntry{}, fmt.Errorf("%q: the protocol is tcp, udp, sctp, icmp or icmpv6", s)
+}
+
+// IsServiceEntry reports whether a reference is an entry ("tcp/22")
+// rather than a service name.
+func IsServiceEntry(ref string) bool { return strings.Contains(ref, "/") }
+
+// NormalizePorts checks a port or range ("22", "1000-2000").
+func NormalizePorts(s string) (string, error) {
+	s = strings.TrimSpace(s)
+	a, b, isRange := strings.Cut(s, "-")
+	lo, err1 := strconv.Atoi(strings.TrimSpace(a))
+	hi := lo
+	var err2 error
+	if isRange {
+		hi, err2 = strconv.Atoi(strings.TrimSpace(b))
+	}
+	if err1 != nil || err2 != nil || lo < 1 || hi > 65535 || hi < lo {
+		return "", fmt.Errorf("%q is not a port (1-65535) or a range (1000-2000)", s)
+	}
+	if lo == hi {
+		return strconv.Itoa(lo), nil
+	}
+	return fmt.Sprintf("%d-%d", lo, hi), nil
+}
+
+// PortRange returns a port spec's bounds.
+func PortRange(s string) (lo, hi int) {
+	a, b, isRange := strings.Cut(s, "-")
+	lo, _ = strconv.Atoi(a)
+	hi = lo
+	if isRange {
+		hi, _ = strconv.Atoi(b)
+	}
+	return lo, hi
+}
+
+// builtinServices are the services every configuration has.
+var builtinServices = map[string][]string{
+	"ssh":        {"tcp/22"},
+	"dns":        {"udp/53", "tcp/53"},
+	"dot":        {"tcp/853"},
+	"http":       {"tcp/80"},
+	"https":      {"tcp/443", "udp/443"},
+	"smtp":       {"tcp/25"},
+	"submission": {"tcp/587"},
+	"smtps":      {"tcp/465"},
+	"imaps":      {"tcp/993"},
+	"ntp":        {"udp/123"},
+	"dhcp":       {"udp/67"},
+	"dhcpv6":     {"udp/547"},
+	"wireguard":  {"udp/51820"},
+	"openvpn":    {"udp/1194"},
+	"ping":       {"icmp/echo-request", "icmpv6/echo-request"},
+	"snmp":       {"udp/161"},
+	"syslog":     {"udp/514"},
+	"mdns":       {"udp/5353"},
+	"rdp":        {"tcp/3389"},
+	"ipsec":      {"udp/500", "udp/4500"},
+}
+
+// BuiltinService returns a built-in service by name (nil if there is
+// none).
+func BuiltinService(name string) *Service {
+	e, ok := builtinServices[strings.ToLower(name)]
+	if !ok {
+		return nil
+	}
+	return &Service{Name: strings.ToLower(name), Entries: append([]string(nil), e...), Builtin: true}
+}
+
+// BuiltinServices lists the built-in services by name.
+func BuiltinServices() []*Service {
+	var out []*Service
+	for n := range builtinServices {
+		out = append(out, BuiltinService(n))
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}

internal/model/settings.go

@@ -0,0 +1,99 @@
+package model
+
+// Settings are the firewall-wide settings: default policies and the rules
+// Housecarl adds by itself ("built-ins"), each of which can be turned off.
+type Settings struct {
+	// Policies for packets no rule decided: accept or drop.
+	InputPolicy   string `json:"input_policy"`
+	ForwardPolicy string `json:"forward_policy"`
+	OutputPolicy  string `json:"output_policy"`
+	// LogPolicyDrops logs (rate-limited) what the input and forward
+	// policies drop.
+	LogPolicyDrops bool `json:"log_policy_drops,omitempty"`
+
+	Builtins Builtins `json:"builtins"`
+
+	// AntispoofV4 and AntispoofV6 are source addresses that can't come
+	// from the internet: dropped on external zones (Builtins.Antispoof),
+	// together with OwnPrefixes.
+	AntispoofV4 []string `json:"antispoof_v4"`
+	AntispoofV6 []string `json:"antispoof_v6"`
+	// OwnPrefixes are your own public prefixes (your delegated IPv6
+	// prefix, say): traffic claiming to come from them can't arrive from
+	// outside.
+	OwnPrefixes []string `json:"own_prefixes,omitempty"`
+
+	// Flowtable devices: forwarded TCP and UDP connections between them
+	// skip the rules once established (fast path). Empty: off. Names
+	// must be exact (no wildcards).
+	FlowtableDevices []string `json:"flowtable_devices,omitempty"`
+
+	// ProbationSeconds is how long a change made over the network has to
+	// be confirmed before it is rolled back (0: the default).
+	ProbationSeconds int `json:"probation_seconds,omitempty"`
+	// ManualConfirm: changes made over the network wait for a person to
+	// confirm them. Otherwise the client that made a change confirms it by
+	// itself once it reaches the API again over a new connection, from the
+	// same address: proof that it can still manage the firewall.
+	ManualConfirm bool `json:"manual_confirm,omitempty"`
+
+	// StatsPerDevice counts forwarded traffic to and from external zones
+	// per inside address (IPv6 per /64), for the statistics' top devices.
+	StatsPerDevice bool `json:"stats_per_device,omitempty"`
+}
+
+// Builtins are rules Housecarl adds itself, shown as "added by Housecarl".
+type Builtins struct {
+	// Loopback accepts everything on lo.
+	Loopback bool `json:"loopback"`
+	// Established accepts packets of established and related connections.
+	Established bool `json:"established"`
+	// DropInvalid drops packets conntrack finds invalid.
+	DropInvalid bool `json:"drop_invalid"`
+	// ICMPv6 accepts the ICMPv6 IPv6 needs (RFC 4890): errors, echo, and
+	// neighbour discovery and MLD on the local link.
+	ICMPv6 bool `json:"icmpv6"`
+	// PingInternal accepts ICMP echo to the firewall from internal zones.
+	PingInternal bool `json:"ping_internal"`
+	// DHCPv6Client accepts DHCPv6 replies on external zones (an ISP's
+	// prefix delegation).
+	DHCPv6Client bool `json:"dhcpv6_client"`
+	// Antispoof drops impossible source addresses arriving on external
+	// zones (AntispoofV4, AntispoofV6, OwnPrefixes).
+	Antispoof bool `json:"antispoof"`
+	// MSSClamp clamps TCP MSS to the route MTU on external zones (needed
+	// with PPPoE).
+	MSSClamp bool `json:"mss_clamp"`
+}
+
+// Policies.
+const (
+	PolicyAccept = "accept"
+	PolicyDrop   = "drop"
+)
+
+// DefaultProbation is how long a network change has to be confirmed.
+const DefaultProbation = 120
+
+// DefaultSettings are the settings of a new database: drop what isn't
+// allowed in and through, allow out, every built-in on.
+func DefaultSettings() Settings {
+	return Settings{
+		InputPolicy:   PolicyDrop,
+		ForwardPolicy: PolicyDrop,
+		OutputPolicy:  PolicyAccept,
+		Builtins: Builtins{Loopback: true, Established: true, DropInvalid: true, ICMPv6: true,
+			PingInternal: true, DHCPv6Client: true, Antispoof: true, MSSClamp: false},
+		AntispoofV4: []string{"0.0.0.0/8", "10.0.0.0/8", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12",
+			"192.0.2.0/24", "192.168.0.0/16", "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4"},
+		AntispoofV6: []string{"::1/128", "::/128", "fc00::/7", "2001:db8::/32", "ff00::/8"},
+	}
+}
+
+// Probation is the probation in seconds, with its default.
+func (s Settings) Probation() int {
+	if s.ProbationSeconds > 0 {
+		return s.ProbationSeconds
+	}
+	return DefaultProbation
+}

internal/model/state.go

@@ -0,0 +1,354 @@
+// Package model holds Housecarl's data: the firewall configuration
+// (State), users and tokens, the history of changes and the audit log.
+package model
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+)
+
+// State is the whole firewall configuration. It is small and changed as a
+// whole (internal/engine); the ruleset is rendered from it
+// (internal/render).
+type State struct {
+	Settings  Settings       `json:"settings"`
+	Zones     []*Zone        `json:"zones"`
+	Sets      []*AddressSet  `json:"sets"`
+	Services  []*Service     `json:"services"`
+	Rules     []*Rule        `json:"rules"`
+	Forwards  []*PortForward `json:"forwards"`
+	SNATs     []*SourceNAT   `json:"snats"`
+	Redirects []*Redirect    `json:"redirects"`
+	BanLists  []*BanList     `json:"ban_lists"`
+}
+
+// Object kinds, for changes and the history.
+const (
+	KindSettings = "settings"
+	KindZone     = "zone"
+	KindSet      = "set"
+	KindService  = "service"
+	KindRule     = "rule"
+	KindForward  = "forward"
+	KindSNAT     = "snat"
+	KindRedirect = "redirect"
+	KindBanList  = "banlist"
+)
+
+// Kinds lists the object kinds (not settings) in display order.
+var Kinds = []string{KindZone, KindSet, KindService, KindRule, KindForward, KindSNAT, KindRedirect, KindBanList}
+
+// Object is anything in the State with an ID.
+type Object interface {
+	GetID() int64
+	Label() string
+}
+
+// NewState is an empty configuration with the default settings.
+func NewState() *State {
+	st := &State{Settings: DefaultSettings()}
+	st.Fill()
+	return st
+}
+
+// Fill replaces nil lists with empty ones (for JSON and comparisons).
+func (st *State) Fill() {
+	if st.Zones == nil {
+		st.Zones = []*Zone{}
+	}
+	if st.Sets == nil {
+		st.Sets = []*AddressSet{}
+	}
+	if st.Services == nil {
+		st.Services = []*Service{}
+	}
+	if st.Rules == nil {
+		st.Rules = []*Rule{}
+	}
+	if st.Forwards == nil {
+		st.Forwards = []*PortForward{}
+	}
+	if st.SNATs == nil {
+		st.SNATs = []*SourceNAT{}
+	}
+	if st.Redirects == nil {
+		st.Redirects = []*Redirect{}
+	}
+	if st.BanLists == nil {
+		st.BanLists = []*BanList{}
+	}
+}
+
+// Objects returns the objects of a kind.
+func (st *State) Objects(kind string) []Object {
+	var out []Object
+	switch kind {
+	case KindZone:
+		for _, o := range st.Zones {
+			out = append(out, o)
+		}
+	case KindSet:
+		for _, o := range st.Sets {
+			out = append(out, o)
+		}
+	case KindService:
+		for _, o := range st.Services {
+			out = append(out, o)
+		}
+	case KindRule:
+		for _, o := range st.Rules {
+			out = append(out, o)
+		}
+	case KindForward:
+		for _, o := range st.Forwards {
+			out = append(out, o)
+		}
+	case KindSNAT:
+		for _, o := range st.SNATs {
+			out = append(out, o)
+		}
+	case KindRedirect:
+		for _, o := range st.Redirects {
+			out = append(out, o)
+		}
+	case KindBanList:
+		for _, o := range st.BanLists {
+			out = append(out, o)
+		}
+	}
+	return out
+}
+
+// Put replaces (or adds, or with data nil deletes) the object of a kind
+// with an ID, from its JSON.
+// Empty reports whether nothing has been configured: no objects of any
+// kind. housecarld applies no ruleset for an empty configuration, so
+// installing and starting it doesn't close the machine.
+func (st *State) Empty() bool {
+	for _, k := range Kinds {
+		if len(st.Objects(k)) > 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func (st *State) Put(kind string, id int64, data json.RawMessage) error {
+	switch kind {
+	case KindZone:
+		return put(&st.Zones, id, data)
+	case KindSet:
+		return put(&st.Sets, id, data)
+	case KindService:
+		return put(&st.Services, id, data)
+	case KindRule:
+		return put(&st.Rules, id, data)
+	case KindForward:
+		return put(&st.Forwards, id, data)
+	case KindSNAT:
+		return put(&st.SNATs, id, data)
+	case KindRedirect:
+		return put(&st.Redirects, id, data)
+	case KindBanList:
+		return put(&st.BanLists, id, data)
+	}
+	return &unknownKind{kind}
+}
+
+type unknownKind struct{ kind string }
+
+func (e *unknownKind) Error() string { return "unknown kind " + e.kind }
+
+func put[T Object](list *[]T, id int64, data json.RawMessage) error {
+	keep := (*list)[:0:0]
+	for _, o := range *list {
+		if o.GetID() != id {
+			keep = append(keep, o)
+		}
+	}
+	if data != nil {
+		var o T
+		if err := json.Unmarshal(data, &o); err != nil {
+			return err
+		}
+		keep = append(keep, o)
+	}
+	*list = keep
+	return nil
+}
+
+// Clone returns a deep copy.
+func (st *State) Clone() *State {
+	b, err := json.Marshal(st)
+	if err != nil {
+		panic(err)
+	}
+	var out State
+	if err := json.Unmarshal(b, &out); err != nil {
+		panic(err)
+	}
+	out.Fill()
+	return &out
+}
+
+// Sort puts the objects in display order: named objects by name, rules by
+// chain and position, NAT rules by position.
+func (st *State) Sort() {
+	sort.SliceStable(st.Zones, func(i, j int) bool { return st.Zones[i].Name < st.Zones[j].Name })
+	sort.SliceStable(st.Sets, func(i, j int) bool { return st.Sets[i].Name < st.Sets[j].Name })
+	sort.SliceStable(st.Services, func(i, j int) bool { return st.Services[i].Name < st.Services[j].Name })
+	sort.SliceStable(st.BanLists, func(i, j int) bool { return st.BanLists[i].Name < st.BanLists[j].Name })
+	sort.SliceStable(st.Rules, func(i, j int) bool {
+		a, b := st.Rules[i], st.Rules[j]
+		if a.Chain != b.Chain {
+			return ChainOrder(a.Chain) < ChainOrder(b.Chain)
+		}
+		return a.Position < b.Position
+	})
+	sort.SliceStable(st.Forwards, func(i, j int) bool { return st.Forwards[i].Position < st.Forwards[j].Position })
+	sort.SliceStable(st.SNATs, func(i, j int) bool { return st.SNATs[i].Position < st.SNATs[j].Position })
+	sort.SliceStable(st.Redirects, func(i, j int) bool { return st.Redirects[i].Position < st.Redirects[j].Position })
+}
+
+// Renumber gives the rules of each chain, and the NAT rules of each kind,
+// positions 1, 2, 3, … in their current order (after an insertion at a
+// fractional position, or a deletion).
+func (st *State) Renumber() {
+	st.Sort()
+	n := map[string]float64{}
+	for _, r := range st.Rules {
+		n[r.Chain]++
+		r.Position = n[r.Chain]
+	}
+	for i, f := range st.Forwards {
+		f.Position = float64(i + 1)
+	}
+	for i, s := range st.SNATs {
+		s.Position = float64(i + 1)
+	}
+	for i, r := range st.Redirects {
+		r.Position = float64(i + 1)
+	}
+}
+
+// Zone finds a zone by name.
+func (st *State) Zone(name string) *Zone {
+	for _, z := range st.Zones {
+		if strings.EqualFold(z.Name, name) {
+			return z
+		}
+	}
+	return nil
+}
+
+// Set finds an address set by name.
+func (st *State) Set(name string) *AddressSet {
+	for _, s := range st.Sets {
+		if strings.EqualFold(s.Name, name) {
+			return s
+		}
+	}
+	return nil
+}
+
+// Service finds a service by name: the configuration's, else the built-in
+// catalog's.
+func (st *State) Service(name string) *Service {
+	for _, s := range st.Services {
+		if strings.EqualFold(s.Name, name) {
+			return s
+		}
+	}
+	return BuiltinService(name)
+}
+
+// ByID finds an object of a kind by ID (nil if there is none).
+func (st *State) ByID(kind string, id int64) Object {
+	for _, o := range st.Objects(kind) {
+		if o.GetID() == id {
+			return o
+		}
+	}
+	return nil
+}
+
+// Find finds an object of a kind by ID or name (zones, sets, services,
+// port forwards, ban lists) or by ID alone (rules, source NAT, redirects).
+func (st *State) Find(kind, ref string) Object {
+	for _, o := range st.Objects(kind) {
+		if itoa(o.GetID()) == ref {
+			return o
+		}
+	}
+	switch kind {
+	case KindZone:
+		if z := st.Zone(ref); z != nil {
+			return z
+		}
+	case KindSet:
+		if s := st.Set(ref); s != nil {
+			return s
+		}
+	case KindService:
+		for _, s := range st.Services {
+			if strings.EqualFold(s.Name, ref) {
+				return s
+			}
+		}
+	case KindForward:
+		for _, f := range st.Forwards {
+			if f.Name != "" && strings.EqualFold(f.Name, ref) {
+				return f
+			}
+		}
+	case KindBanList:
+		if b := st.BanList(ref); b != nil {
+			return b
+		}
+	}
+	return nil
+}
+
+// BanList finds a ban list by name.
+func (st *State) BanList(name string) *BanList {
+	for _, b := range st.BanLists {
+		if strings.EqualFold(b.Name, name) {
+			return b
+		}
+	}
+	return nil
+}
+
+func itoa(n int64) string {
+	b, _ := json.Marshal(n)
+	return string(b)
+}
+
+// ExternalInterfaces lists the interfaces of the external zones.
+func (st *State) ExternalInterfaces() []string {
+	return st.zoneInterfaces(func(z *Zone) bool { return z.External })
+}
+
+// InternalInterfaces lists the interfaces of the internal zones.
+func (st *State) InternalInterfaces() []string {
+	return st.zoneInterfaces(func(z *Zone) bool { return !z.External })
+}
+
+func (st *State) zoneInterfaces(keep func(*Zone) bool) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, z := range st.Zones {
+		if !keep(z) {
+			continue
+		}
+		for _, i := range z.Interfaces {
+			if !seen[i] {
+				seen[i] = true
+				out = append(out, i)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}

internal/names/names.go

@@ -0,0 +1,152 @@
+// Package names finds the names of addresses by reverse lookups, for
+// showing them beside the addresses (connections, the log, quotas,
+// traffic per device). Names are cached and never stored: statistics stay
+// keyed by address.
+package names
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/netip"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Lookups running at once, and the most names kept.
+const (
+	parallel = 16
+	maxCache = 20000
+)
+
+// Resolver looks up and caches addresses' names.
+type Resolver struct {
+	lookup func(ctx context.Context, addr string) ([]string, error)
+	// ttl is how long a name (or its absence) is kept; a lookup that
+	// failed (a timeout, a server error) is tried again after retry.
+	ttl, retry time.Duration
+	timeout    time.Duration
+	now        func() time.Time
+
+	mu      sync.Mutex
+	cache   map[netip.Addr]entry
+	pending map[netip.Addr]chan struct{}
+	sem     chan struct{}
+}
+
+type entry struct {
+	name    string
+	expires time.Time
+}
+
+// New makes a resolver asking server ("10.10.0.1:53"; "" for the
+// system's, /etc/hosts then /etc/resolv.conf), keeping names for ttl.
+func New(server string, ttl time.Duration) *Resolver {
+	res := &net.Resolver{PreferGo: true}
+	if server != "" {
+		res.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
+			var d net.Dialer
+			return d.DialContext(ctx, network, server)
+		}
+	}
+	return NewFunc(res.LookupAddr, ttl)
+}
+
+// NewFunc makes a resolver with its own lookup (tests).
+func NewFunc(lookup func(ctx context.Context, addr string) ([]string, error), ttl time.Duration) *Resolver {
+	return &Resolver{lookup: lookup, ttl: ttl, retry: time.Minute, timeout: 2 * time.Second, now: time.Now,
+		cache: map[netip.Addr]entry{}, pending: map[netip.Addr]chan struct{}{}, sem: make(chan struct{}, parallel)}
+}
+
+// Names returns the names of addrs: those cached, and those looked up
+// within wait. Lookups still running then carry on and are cached for
+// the next call. An address without a name is left out.
+func (r *Resolver) Names(ctx context.Context, addrs []netip.Addr, wait time.Duration) map[netip.Addr]string {
+	out := map[netip.Addr]string{}
+	var waiting []netip.Addr
+	var done []chan struct{}
+	now := r.now()
+	r.mu.Lock()
+	for _, a := range addrs {
+		if e, ok := r.cache[a]; ok && now.Before(e.expires) {
+			if e.name != "" {
+				out[a] = e.name
+			}
+			continue
+		}
+		ch, ok := r.pending[a]
+		if !ok {
+			ch = make(chan struct{})
+			r.pending[a] = ch
+			go r.resolve(a, ch)
+		}
+		waiting = append(waiting, a)
+		done = append(done, ch)
+	}
+	r.mu.Unlock()
+	if len(waiting) == 0 {
+		return out
+	}
+	t := time.NewTimer(wait)
+	defer t.Stop()
+	for _, ch := range done {
+		select {
+		case <-ch:
+		case <-t.C:
+			return r.collect(out, waiting)
+		case <-ctx.Done():
+			return r.collect(out, waiting)
+		}
+	}
+	return r.collect(out, waiting)
+}
+
+// collect adds what the lookups found for addrs.
+func (r *Resolver) collect(out map[netip.Addr]string, addrs []netip.Addr) map[netip.Addr]string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, a := range addrs {
+		if e, ok := r.cache[a]; ok && e.name != "" {
+			out[a] = e.name
+		}
+	}
+	return out
+}
+
+func (r *Resolver) resolve(a netip.Addr, done chan struct{}) {
+	r.sem <- struct{}{}
+	ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+	ns, err := r.lookup(ctx, a.String())
+	cancel()
+	<-r.sem
+	e := entry{expires: r.now().Add(r.ttl)}
+	var dnsErr *net.DNSError
+	switch {
+	case err == nil && len(ns) > 0:
+		e.name = strings.TrimSuffix(ns[0], ".")
+	case err != nil && !(errors.As(err, &dnsErr) && dnsErr.IsNotFound):
+		e.expires = r.now().Add(r.retry)
+	}
+	r.mu.Lock()
+	if len(r.cache) >= maxCache {
+		r.prune()
+	}
+	r.cache[a] = e
+	delete(r.pending, a)
+	r.mu.Unlock()
+	close(done)
+}
+
+// prune drops expired names, or all of them when none has expired.
+func (r *Resolver) prune() {
+	now := r.now()
+	for a, e := range r.cache {
+		if !now.Before(e.expires) {
+			delete(r.cache, a)
+		}
+	}
+	if len(r.cache) >= maxCache {
+		clear(r.cache)
+	}
+}

internal/names/names_test.go

@@ -0,0 +1,57 @@
+package names
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/netip"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestNames(t *testing.T) {
+	var calls atomic.Int32
+	slow := make(chan struct{})
+	r := NewFunc(func(ctx context.Context, addr string) ([]string, error) {
+		calls.Add(1)
+		switch addr {
+		case "10.0.0.1":
+			return []string{"nas.example.net."}, nil
+		case "10.0.0.2":
+			return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true}
+		case "10.0.0.3":
+			return nil, errors.New("server failure")
+		case "10.0.0.4":
+			<-slow
+			return []string{"slow.example.net."}, nil
+		}
+		return nil, nil
+	}, time.Hour)
+	a := netip.MustParseAddr
+	all := []netip.Addr{a("10.0.0.1"), a("10.0.0.2"), a("10.0.0.3"), a("10.0.0.4")}
+
+	got := r.Names(context.Background(), all, 200*time.Millisecond)
+	if len(got) != 1 || got[a("10.0.0.1")] != "nas.example.net" {
+		t.Fatalf("first call: %v", got)
+	}
+	// The slow lookup finishes later and is there next time; the others
+	// come from the cache, the failure too until it is retried.
+	close(slow)
+	time.Sleep(50 * time.Millisecond)
+	got = r.Names(context.Background(), all, 200*time.Millisecond)
+	if len(got) != 2 || got[a("10.0.0.4")] != "slow.example.net" {
+		t.Fatalf("second call: %v", got)
+	}
+	if n := calls.Load(); n != 4 {
+		t.Fatalf("%d lookups, want 4 (cached)", n)
+	}
+
+	// A failure is tried again after retry, a missing name after ttl.
+	now := time.Now().Add(2 * time.Minute)
+	r.now = func() time.Time { return now }
+	r.Names(context.Background(), all, 200*time.Millisecond)
+	if n := calls.Load(); n != 5 {
+		t.Fatalf("%d lookups, want 5 (the failure again)", n)
+	}
+}