Add changesets, bandwidth limits and quotas; generic examples

Changesets, as in Jarl: configuration changes are operations that can be
staged in a changeset, checked as they are added (validation and nft -c),
reviewed with the diff of the ruleset, and applied together as one
history entry, then on probation like any change made over the network.
The web UI collects edits for review by default (pending rows, a bar with
Review and Apply now, a Changes page, a switch for live edits); hc stages
with -C ID or HOUSECARL_CHANGESET (checked like JARL_CHANGESET) and has
hc changeset create/list/show/diff/apply/discard/remove. Undo, import and
takeover refuse a changeset. --dry-run shows the ruleset's diff.

Bandwidth and quotas: rate limits in bytes, per source or destination;
quotas per rule or per address that reset daily, weekly, monthly on a
day, or by hand, with their usage carried across reloads, saved for
reboots, shown by hc quotas and on the rules page, and reset with hc rule
reset-quota. Rules that count traffic are checked before established
connections are accepted (so they see every packet) and can't accept.
Per-address limits group IPv6 by /64 by default (--group-v4/-v6).
Per-source rate meters use update. The importer turns meters, byte rates
and quotas into these fields.

The examples in the web UI, hc's help and the docs use documentation
addresses and generic names instead of the maintainer's network.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/h_config.go

@@ -7,6 +7,8 @@ import (
 	"sort"
 	"strings"
 
+	udiff "github.com/aymanbagabas/go-udiff"
+
 	"housecarl/internal/api"
 	"housecarl/internal/auth"
 	"housecarl/internal/engine"
@@ -14,26 +16,46 @@ import (
 	"housecarl/internal/store"
 )
 
-// change applies an edit through the engine and answers it; object picks
-// what to return from the new state. A network change starts (or joins)
-// the probation.
-func (s *Server) change(w http.ResponseWriter, r *http.Request, p *auth.Principal, description string, edit engine.Edit, object func(st *model.State) any) error {
+// change makes an operation: at once through the engine, or, with a
+// changeset named (?changeset=ID or X-Housecarl-Changeset), staged in it.
+// A network change starts (or joins) the probation.
+func (s *Server) change(w http.ResponseWriter, r *http.Request, p *auth.Principal, op model.Op) error {
 	if err := require(p, auth.ResConfig, true); err != nil {
 		return err
 	}
-	res, err := s.eng.Apply(r.Context(), p, description, engine.Options{DryRun: queryBool(r, "dry_run"), Via: via(r, p)}, edit)
+	if op.Op == model.OpCreate && op.ID == 0 {
+		ids, err := s.store.AllocateIDs(r.Context(), 1)
+		if err != nil {
+			return err
+		}
+		op.ID = ids[0]
+	}
+	csID, err := changesetParam(r)
+	if err != nil {
+		return err
+	}
+	if csID != 0 {
+		return s.stage(w, r, p, csID, op)
+	}
+	var id int64
+	res, err := s.eng.Apply(r.Context(), p, "", engine.Options{DryRun: queryBool(r, "dry_run"), Via: via(r, p)}, func(st *model.State, _ func() int64) error {
+		var err error
+		id, err = applyOp(st, &op)
+		return err
+	})
 	if err != nil {
 		s.writeErr(w, r, err, res)
 		return nil
 	}
 	s.afterChange(r.Context(), p, res)
-	// POST to a collection creates; POST to an object (move) changes it.
-	s.respondChange(w, r, res, object, r.Method == http.MethodPost && r.PathValue("ref") == "")
+	writeJSON(w, s.mutationStatus(res, op.Op == model.OpCreate), s.mutation(res, objectOf(&op, id)))
 	return nil
 }
 
-// respondChange answers a change: 201 when it created something.
-func (s *Server) respondChange(w http.ResponseWriter, r *http.Request, res *engine.Result, object func(st *model.State) any, creates bool) {
+// mutation is the answer to a change: what changed, warnings, the object
+// (as it is after the change), the probation, and for a dry run the
+// ruleset's diff.
+func (s *Server) mutation(res *engine.Result, object func(st *model.State) any) api.MutationResponse {
 	out := api.MutationResponse{Applied: res.Applied, DryRun: res.DryRun, HistoryID: res.HistoryID, Changes: res.Changes, Warnings: res.Warnings}
 	if out.Changes == nil {
 		out.Changes = []model.Change{}
@@ -41,16 +63,37 @@ func (s *Server) respondChange(w http.ResponseWriter, r *http.Request, res *engi
 	if pr := s.probationStatus(); res.Applied && (len(pr.Pending) > 0 || pr.Takeovers > 0) {
 		out.ConfirmBy = &pr.Deadline
 	}
-	if object != nil && (res.Applied || res.DryRun) {
-		if v := object(s.eng.State()); v != nil {
+	if object != nil && res.After != nil && (res.Applied || res.DryRun) {
+		if v := object(res.After); v != nil {
 			out.Object, _ = json.Marshal(v)
 		}
 	}
-	status := http.StatusOK
-	if res.Applied && creates {
-		status = http.StatusCreated
+	if res.DryRun && len(res.Changes) > 0 {
+		out.Diff = s.rulesetDiff(res.Before, res.After)
 	}
-	writeJSON(w, status, out)
+	return out
+}
+
+// mutationStatus is 201 for an applied create, else 200.
+func (s *Server) mutationStatus(res *engine.Result, creates bool) int {
+	if res.Applied && creates {
+		return http.StatusCreated
+	}
+	return http.StatusOK
+}
+
+// respondChange answers a change made without an operation (undo).
+func (s *Server) respondChange(w http.ResponseWriter, r *http.Request, res *engine.Result, object func(st *model.State) any, creates bool) {
+	writeJSON(w, s.mutationStatus(res, creates), s.mutation(res, object))
+}
+
+// rulesetDiff is the unified diff of the rulesets two configurations
+// make: the file housecarl-ruleset.service loads, now and after.
+func (s *Server) rulesetDiff(before, after *model.State) string {
+	if before == nil || after == nil {
+		return ""
+	}
+	return udiff.Unified("ruleset.nft (now)", "ruleset.nft (after)", s.fw.Render(before).Text, s.fw.Render(after).Text)
 }
 
 // kindOf maps the API's plural path names to kinds.
@@ -84,16 +127,29 @@ func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, p *auth.Pri
 	if err != nil {
 		return err
 	}
-	st := s.eng.State()
+	cur, st, draft, err := s.viewState(w, r)
+	if err != nil {
+		return err
+	}
 	out := []any{}
-	q := r.URL.Query()
-	for _, o := range st.Objects(kind) {
-		if c := q.Get("chain"); c != "" {
-			if rule, ok := o.(*model.Rule); ok && rule.Chain != c {
+	c := r.URL.Query().Get("chain")
+	if draft {
+		// Marked new, changed or deleted.
+		for _, o := range withPending(cur, st, kind) {
+			if c != "" && kind == model.KindRule && o.(map[string]any)["chain"] != c {
 				continue
 			}
+			out = append(out, o)
+		}
+	} else {
+		for _, o := range st.Objects(kind) {
+			if c != "" {
+				if rule, ok := o.(*model.Rule); ok && rule.Chain != c {
+					continue
+				}
+			}
+			out = append(out, o)
 		}
-		out = append(out, o)
 	}
 	// Services: the built-in ones too, unless overridden.
 	if kind == model.KindService && !queryBool(r, "own") {
@@ -102,9 +158,14 @@ func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, p *auth.Pri
 				out = append(out, b)
 			}
 		}
-		sort.SliceStable(out, func(i, j int) bool {
-			return out[i].(*model.Service).Name < out[j].(*model.Service).Name
-		})
+		name := func(x any) string {
+			if m, ok := x.(map[string]any); ok {
+				n, _ := m["name"].(string)
+				return n
+			}
+			return x.(*model.Service).Name
+		}
+		sort.SliceStable(out, func(i, j int) bool { return name(out[i]) < name(out[j]) })
 	}
 	writeJSON(w, http.StatusOK, out)
 	return nil
@@ -118,7 +179,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request, p *auth.Princ
 	if err != nil {
 		return err
 	}
-	st := s.eng.State()
+	_, st, _, err := s.viewState(w, r)
+	if err != nil {
+		return err
+	}
 	o, err := findObject(st, kind, r.PathValue("ref"))
 	if err != nil && kind == model.KindService {
 		if b := model.BuiltinService(r.PathValue("ref")); b != nil {
@@ -235,29 +299,11 @@ func (s *Server) createObject(w http.ResponseWriter, r *http.Request, p *auth.Pr
 	if err := decodeJSON(r, &raw); err != nil {
 		return err
 	}
-	o, err := decodeObject(kind, raw)
-	if err != nil {
+	// Refuse a bad object now, not when it is applied.
+	if _, err := decodeObject(kind, raw); err != nil {
 		return err
 	}
-	if svc, ok := o.(*model.Service); ok {
-		svc.Builtin = false
-	}
-	if rule, ok := o.(*model.Rule); ok && rule.Action == "" {
-		rule.Action = model.ActionAccept
-	}
-	if f, ok := o.(*model.PortForward); ok && len(f.Protocols) == 0 {
-		f.Protocols = []string{"tcp"}
-	}
-	var id int64
-	return s.change(w, r, p, "", func(st *model.State, newID func() int64) error {
-		id = newID()
-		setID(o, id)
-		if pos := position(o); pos != nil && *pos == 0 {
-			*pos = lastPosition(st, kind, chainOf(o))
-		}
-		data, _ := json.Marshal(o)
-		return st.Put(kind, id, data)
-	}, func(st *model.State) any { return st.ByID(kind, id) })
+	return s.change(w, r, p, model.Op{Op: model.OpCreate, Kind: kind, Data: raw})
 }
 
 func (s *Server) updateObject(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
@@ -269,34 +315,8 @@ func (s *Server) updateObject(w http.ResponseWriter, r *http.Request, p *auth.Pr
 	if err := decodeJSON(r, &patch); err != nil {
 		return err
 	}
-	delete(patch, "id")
-	var id int64
-	return s.change(w, r, p, "", func(st *model.State, _ func() int64) error {
-		o, err := findObject(st, kind, r.PathValue("ref"))
-		if err != nil {
-			return err
-		}
-		id = o.GetID()
-		// The patch's fields over the object's.
-		var cur map[string]json.RawMessage
-		b, _ := json.Marshal(o)
-		json.Unmarshal(b, &cur)
-		for k, v := range patch {
-			if string(v) == "null" {
-				delete(cur, k)
-			} else {
-				cur[k] = v
-			}
-		}
-		merged, _ := json.Marshal(cur)
-		next, err := decodeObject(kind, merged)
-		if err != nil {
-			return err
-		}
-		setID(next, id)
-		data, _ := json.Marshal(next)
-		return st.Put(kind, id, data)
-	}, func(st *model.State) any { return st.ByID(kind, id) })
+	data, _ := json.Marshal(patch)
+	return s.change(w, r, p, model.Op{Op: model.OpUpdate, Kind: kind, Ref: r.PathValue("ref"), Data: data})
 }
 
 func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
@@ -304,16 +324,7 @@ func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, p *auth.Pr
 	if err != nil {
 		return err
 	}
-	return s.change(w, r, p, "", func(st *model.State, _ func() int64) error {
-		o, err := findObject(st, kind, r.PathValue("ref"))
-		if err != nil {
-			return err
-		}
-		if used := usedBy(st, kind, o); used != "" {
-			return conflict("%s %s is used by %s; change that first", kind, o.Label(), used)
-		}
-		return st.Put(kind, o.GetID(), nil)
-	}, nil)
+	return s.change(w, r, p, model.Op{Op: model.OpDelete, Kind: kind, Ref: r.PathValue("ref")})
 }
 
 // usedBy names an object that refers to o (zones, sets and services are
@@ -364,8 +375,7 @@ func usedBy(st *model.State, kind string, o model.Object) string {
 	return ""
 }
 
-// moveObject reorders a rule or NAT rule: before or after another, or to
-// the start or end; a rule can change chain at the same time.
+// moveObject reorders a rule or NAT rule (moveOp).
 func (s *Server) moveObject(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
 	kind, err := pathKind(r)
 	if err != nil {
@@ -375,72 +385,8 @@ func (s *Server) moveObject(w http.ResponseWriter, r *http.Request, p *auth.Prin
 	if err := decodeJSON(r, &in); err != nil {
 		return err
 	}
-	var id int64
-	return s.change(w, r, p, "", func(st *model.State, _ func() int64) error {
-		o, err := findObject(st, kind, r.PathValue("ref"))
-		if err != nil {
-			return err
-		}
-		id = o.GetID()
-		pos := position(o)
-		if pos == nil {
-			return badRequest("only rules, port forwards, source NAT rules and redirects have an order")
-		}
-		if rule, ok := o.(*model.Rule); ok && in.Chain != "" {
-			rule.Chain = in.Chain
-		}
-		chain := chainOf(o)
-		// The others in order.
-		var others []model.Object
-		for _, x := range st.Objects(kind) {
-			if x.GetID() != id && chainOf(x) == chain {
-				others = append(others, x)
-			}
-		}
-		sort.SliceStable(others, func(i, j int) bool { return *position(others[i]) < *position(others[j]) })
-		place := func(i int) { // before others[i] (i == len: at the end)
-			switch {
-			case len(others) == 0:
-				*pos = 1
-			case i == 0:
-				*pos = *position(others[0]) - 1
-			case i == len(others):
-				*pos = *position(others[len(others)-1]) + 1
-			default:
-				*pos = (*position(others[i-1]) + *position(others[i])) / 2
-			}
-		}
-		index := func(ref int64) (int, error) {
-			for i, x := range others {
-				if x.GetID() == ref {
-					return i, nil
-				}
-			}
-			return 0, badRequest("%d isn't another %s of the same chain", ref, kind)
-		}
-		switch {
-		case in.Before != 0:
-			i, err := index(in.Before)
-			if err != nil {
-				return err
-			}
-			place(i)
-		case in.After != 0:
-			i, err := index(in.After)
-			if err != nil {
-				return err
-			}
-			place(i + 1)
-		case in.To == "start":
-			place(0)
-		case in.To == "end" || in.Chain != "":
-			place(len(others))
-		default:
-			return badRequest("say where: before, after, or to start or end")
-		}
-		data, _ := json.Marshal(o)
-		return st.Put(kind, id, data)
-	}, func(st *model.State) any { return st.ByID(kind, id) })
+	data, _ := json.Marshal(in)
+	return s.change(w, r, p, model.Op{Op: model.OpMove, Kind: kind, Ref: r.PathValue("ref"), Data: data})
 }
 
 // ---- settings
@@ -449,7 +395,11 @@ func (s *Server) getSettings(w http.ResponseWriter, r *http.Request, p *auth.Pri
 	if err := require(p, auth.ResConfig, false); err != nil {
 		return err
 	}
-	writeJSON(w, http.StatusOK, s.eng.State().Settings)
+	_, st, _, err := s.viewState(w, r)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, st.Settings)
 	return nil
 }
 
@@ -458,35 +408,8 @@ func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request, p *auth.
 	if err := decodeJSON(r, &patch); err != nil {
 		return err
 	}
-	return s.change(w, r, p, "", func(st *model.State, _ func() int64) error {
-		var cur map[string]json.RawMessage
-		b, _ := json.Marshal(st.Settings)
-		json.Unmarshal(b, &cur)
-		for k, v := range patch {
-			// Built-ins merge one by one.
-			if k == "builtins" {
-				var cb, pb map[string]json.RawMessage
-				json.Unmarshal(cur[k], &cb)
-				if err := json.Unmarshal(v, &pb); err != nil {
-					return badRequest("builtins: %v", err)
-				}
-				for bk, bv := range pb {
-					cb[bk] = bv
-				}
-				v, _ = json.Marshal(cb)
-			}
-			cur[k] = v
-		}
-		merged, _ := json.Marshal(cur)
-		var next model.Settings
-		dec := json.NewDecoder(strings.NewReader(string(merged)))
-		dec.DisallowUnknownFields()
-		if err := dec.Decode(&next); err != nil {
-			return badRequest("settings: %v", err)
-		}
-		st.Settings = next
-		return nil
-	}, func(st *model.State) any { return st.Settings })
+	data, _ := json.Marshal(patch)
+	return s.change(w, r, p, model.Op{Op: model.OpSettings, Data: data})
 }
 
 // ---- the ruleset
@@ -495,7 +418,11 @@ func (s *Server) ruleset(w http.ResponseWriter, r *http.Request, p *auth.Princip
 	if err := require(p, auth.ResConfig, false); err != nil {
 		return err
 	}
-	rs := s.fw.Render(s.eng.State())
+	_, st, _, err := s.viewState(w, r)
+	if err != nil {
+		return err
+	}
+	rs := s.fw.Render(st)
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	w.Write([]byte(rs.Text))
 	return nil
@@ -538,6 +465,9 @@ func (s *Server) revertHistory(w http.ResponseWriter, r *http.Request, p *auth.P
 	if err := require(p, auth.ResConfig, true); err != nil {
 		return err
 	}
+	if err := notStaged(r, "an undo"); err != nil {
+		return err
+	}
 	id, err := pathID(r, "id")
 	if err != nil {
 		return err