Add blocking: blocklists and block/allow rules with BIND response policy zones

jarld downloads blocklists (Adblock, hosts or plain domains), checks them
daily and reloads named only when a list changed; your own block and
allow rules take precedence, and the most specific rule wins. Lists and
rules apply per view (default: every view that offers recursion). Web UI
Blocking page with name checks, jarl blocklist and jarl block, the
/blocking API, blocked counts in the statistics. Database schema 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/blocking_test.go

@@ -0,0 +1,95 @@
+package server
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestBlockingAPI(t *testing.T) {
+	e := newAPIEnv(t)
+	lists := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/missing" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Write([]byte("||ads.example^\n0.0.0.0 tracker.example\n"))
+	}))
+	defer lists.Close()
+
+	b := e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": lists.URL + "/list.txt"}, 201)["object"].(map[string]any)
+	if b["name"] != "ads" || b["state"] != "waiting" {
+		t.Errorf("new list = %v", b)
+	}
+	got := e.do("POST", "/blocking/lists/ads/refresh", e.token, nil, 200)
+	st, _ := got["status"].(map[string]any)
+	if got["state"] != "ok" || st["entries"] != 2.0 || st["records"] != 3.0 {
+		t.Errorf("after refresh = %v", got)
+	}
+	e.do("POST", "/blocking/rules", e.token, map[string]any{"name": "Good.Ads.Example.", "action": "allow"}, 201)
+	r := e.do("POST", "/blocking/rules", e.token, map[string]any{"name": "more.example", "exact": true}, 201)["object"].(map[string]any)
+	if r["name"] != "more.example" || r["action"] != "block" || r["exact"] != true {
+		t.Errorf("new rule = %v", r)
+	}
+	for name, want := range map[string]string{"x.ads.example": "blocked", "a.good.ads.example": "allowed", "more.example": "blocked", "sub.more.example": "none"} {
+		m := e.list("/blocking/check?name="+name, e.token)
+		if len(m) != 1 || m[0]["result"] != want {
+			t.Errorf("check %s = %v, want %s", name, m, want)
+		}
+	}
+	if _, status := e.req("GET", "/blocking/check?name=com", e.token, nil); status != 400 {
+		t.Errorf("check of a TLD = %d", status)
+	}
+
+	// Staged changes show as pending.
+	cs := itoa(int64(e.do("POST", "/changesets", e.token, map[string]any{"description": "more"}, 201)["id"].(float64)))
+	e.do("POST", "/blocking/rules?changeset="+cs, e.token, map[string]any{"name": "staged.example"}, 202)
+	e.do("PATCH", "/blocking/lists/ads?changeset="+cs, e.token, map[string]any{"disabled": true}, 202)
+	pending := map[string]any{}
+	for _, x := range e.list("/blocking/rules?changeset="+cs, e.token) {
+		pending[x["name"].(string)] = x["pending"]
+	}
+	if pending["staged.example"] != "add" || pending["more.example"] != nil {
+		t.Errorf("pending rules = %v", pending)
+	}
+	if l := e.list("/blocking/lists?changeset="+cs, e.token); len(l) != 1 || l[0]["pending"] != "change" || l[0]["state"] != "disabled" {
+		t.Errorf("pending list = %v", l)
+	}
+	e.do("POST", "/changesets/"+cs+"/discard", e.token, nil, 204)
+
+	// Delete rules by name; a missing one is a 404 unless missing_ok.
+	e.do("DELETE", "/blocking/rules?name=more.example", e.token, nil, 200)
+	if _, status := e.req("DELETE", "/blocking/rules?name=more.example", e.token, nil); status != 404 {
+		t.Errorf("deleting a missing rule = %d", status)
+	}
+	e.do("DELETE", "/blocking/rules?name=more.example&missing_ok=1", e.token, nil, 200)
+
+	// Readers see blocking but can't change it.
+	e.do("POST", "/users", e.token, map[string]any{"username": "reader", "password": "reader-password-123", "grant_strings": []string{"blocking:read"}}, 201)
+	var readerID int64
+	for _, u := range e.list("/users", e.token) {
+		if u["username"] == "reader" {
+			readerID = int64(u["id"].(float64))
+		}
+	}
+	rt := e.do("POST", "/users/"+itoa(readerID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string)
+	if l := e.list("/blocking/lists", rt); len(l) != 1 {
+		t.Errorf("reader's lists = %v", l)
+	}
+	for _, c := range []struct{ method, path string }{{"POST", "/blocking/rules"}, {"POST", "/blocking/lists/ads/refresh"}, {"DELETE", "/blocking/lists/ads"}} {
+		if _, status := e.req(c.method, c.path, rt, map[string]any{"name": "x.example"}); status != 403 {
+			t.Errorf("reader %s %s = %d", c.method, c.path, status)
+		}
+	}
+	// A list that can't be downloaded shows on the dashboard.
+	e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "gone", "url": lists.URL + "/missing"}, 201)
+	if l := e.do("POST", "/blocking/lists/gone/refresh", e.token, nil, 200); l["state"] != "error" {
+		t.Errorf("failed list = %v", l)
+	}
+	errs, _ := e.do("GET", "/status", e.token, nil, 200)["blocklist_errors"].([]any)
+	if len(errs) != 1 || errs[0].(map[string]any)["name"] != "gone" {
+		t.Errorf("status blocklist_errors = %v", errs)
+	}
+	e.do("DELETE", "/blocking/lists/gone", e.token, nil, 200)
+	e.do("DELETE", "/blocking/lists/ads", e.token, nil, 200)
+}

internal/server/h_blocking.go

@@ -0,0 +1,368 @@
+package server
+
+import (
+	"context"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"jarl/internal/api"
+	"jarl/internal/auth"
+	"jarl/internal/blocking"
+	"jarl/internal/engine"
+	"jarl/internal/model"
+)
+
+// ---- blocking (docs/blocking.md)
+
+// bytesPerRecord is what named 9.20 needs per policy zone record and view,
+// measured with lists of a few hundred thousand entries.
+const bytesPerRecord = 630
+
+func viewNames(st *model.State, ids []int64) []string {
+	out := []string{}
+	for _, id := range ids {
+		if v := st.Views[id]; v != nil {
+			out = append(out, v.Name)
+		}
+	}
+	return out
+}
+
+func presentBlocklist(st *model.State, b *model.Blocklist, status *model.BlocklistStatus) api.Blocklist {
+	views := st.BlockingViews(b.ViewIDs)
+	out := api.Blocklist{Blocklist: *b, Views: viewNames(st, views), AutoViews: len(b.ViewIDs) == 0 && len(st.Views) > 0, Status: status}
+	switch {
+	case b.Disabled:
+		out.State = "disabled"
+	case len(views) == 0:
+		out.State = "idle"
+	case status != nil && status.Error != "":
+		out.State = "error"
+	case status == nil || status.UpdatedAt == nil:
+		out.State = "waiting"
+	default:
+		out.State = "ok"
+	}
+	if status != nil && !b.Disabled {
+		out.MemoryMB = int((int64(status.Records)*bytesPerRecord*int64(len(views)) + 1<<19) >> 20)
+	}
+	return out
+}
+
+func presentBlockRule(st *model.State, r *model.BlockRule) api.BlockRule {
+	return api.BlockRule{BlockRule: *r, Views: viewNames(st, st.BlockingViews(r.ViewIDs)), AutoViews: len(r.ViewIDs) == 0 && len(st.Views) > 0}
+}
+
+func requireBlocking(p *auth.Principal) error {
+	return require(p, auth.Action{Resource: auth.ResBlocking})
+}
+
+func (s *Server) listBlocklists(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := requireBlocking(p); err != nil {
+		return err
+	}
+	st, base, err := s.states(r)
+	if err != nil {
+		return err
+	}
+	statuses, err := s.store.BlocklistStatuses(r.Context())
+	if err != nil {
+		return err
+	}
+	out := []api.Blocklist{}
+	for _, b := range st.SortedBlocklists() {
+		o := presentBlocklist(st, b, statuses[b.ID])
+		o.Pending = pending(base, base != nil && base.Blocklists[b.ID] == nil, base != nil && base.Blocklists[b.ID] != nil && base.Blocklists[b.ID].Version != b.Version)
+		out = append(out, o)
+	}
+	if base != nil {
+		for _, b := range base.SortedBlocklists() {
+			if st.Blocklists[b.ID] == nil {
+				o := presentBlocklist(base, b, statuses[b.ID])
+				o.Pending = "delete"
+				out = append(out, o)
+			}
+		}
+	}
+	writeJSON(w, http.StatusOK, out)
+	return nil
+}
+
+// pending returns the pending marker of an object listed with a changeset.
+func pending(base *model.State, added, changed bool) string {
+	switch {
+	case base == nil:
+		return ""
+	case added:
+		return "add"
+	case changed:
+		return "change"
+	}
+	return ""
+}
+
+func (s *Server) blocklistByPath(r *http.Request, st *model.State) (*model.Blocklist, error) {
+	ref := r.PathValue("id")
+	var b *model.Blocklist
+	if id, err := strconv.ParseInt(ref, 10, 64); err == nil {
+		b = st.Blocklists[id]
+	} else {
+		b = st.BlocklistByName(ref)
+	}
+	if b == nil {
+		return nil, notFound("blocklist %s not found", ref)
+	}
+	return b, nil
+}
+
+func (s *Server) blocklistObject(ctx context.Context) func(*model.State, int64) any {
+	return func(st *model.State, id int64) any {
+		b := st.Blocklists[id]
+		if b == nil {
+			return nil
+		}
+		statuses, _ := s.store.BlocklistStatuses(ctx)
+		return presentBlocklist(st, b, statuses[id])
+	}
+}
+
+func (s *Server) getBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := requireBlocking(p); err != nil {
+		return err
+	}
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	b, err := s.blocklistByPath(r, st)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, http.StatusOK, s.blocklistObject(r.Context())(st, b.ID))
+	return nil
+}
+
+func (s *Server) createBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	body, err := readBody(r)
+	if err != nil {
+		return err
+	}
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistCreate, 0, body)}, s.blocklistObject(r.Context()))
+}
+
+func (s *Server) updateBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	b, err := s.blocklistByPath(r, st)
+	if err != nil {
+		return err
+	}
+	body, err := readBody(r)
+	if err != nil {
+		return err
+	}
+	id := b.ID
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistUpdate, id, body)}, func(st *model.State, _ int64) any { return s.blocklistObject(r.Context())(st, id) })
+}
+
+func (s *Server) deleteBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	b, err := s.blocklistByPath(r, st)
+	if err != nil {
+		return err
+	}
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistDelete, b.ID, nil)}, nil)
+}
+
+// refreshBlocklist: POST /blocking/lists/{id}/refresh[?force=1] downloads
+// the list now and waits for the result.
+func (s *Server) refreshBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := require(p, auth.Action{Resource: auth.ResBlocking, Write: true}); err != nil {
+		return err
+	}
+	st, err := s.store.LoadState(r.Context())
+	if err != nil {
+		return err
+	}
+	b, err := s.blocklistByPath(r, st)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
+	defer cancel()
+	status, rerr := s.eng.RefreshBlocklist(ctx, b.ID, queryBool(r, "force"))
+	if status == nil {
+		return rerr
+	}
+	writeJSON(w, http.StatusOK, presentBlocklist(st, b, status))
+	return nil
+}
+
+func (s *Server) listBlockRules(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := requireBlocking(p); err != nil {
+		return err
+	}
+	st, base, err := s.states(r)
+	if err != nil {
+		return err
+	}
+	out := []api.BlockRule{}
+	for _, rule := range st.SortedBlockRules() {
+		o := presentBlockRule(st, rule)
+		o.Pending = pending(base, base != nil && base.BlockRules[rule.ID] == nil, base != nil && base.BlockRules[rule.ID] != nil && base.BlockRules[rule.ID].Version != rule.Version)
+		out = append(out, o)
+	}
+	if base != nil {
+		for _, rule := range base.SortedBlockRules() {
+			if st.BlockRules[rule.ID] == nil {
+				o := presentBlockRule(base, rule)
+				o.Pending = "delete"
+				out = append(out, o)
+			}
+		}
+	}
+	writeJSON(w, http.StatusOK, out)
+	return nil
+}
+
+func ruleObject(st *model.State, id int64) any {
+	if rule := st.BlockRules[id]; rule != nil {
+		return presentBlockRule(st, rule)
+	}
+	return nil
+}
+
+func (s *Server) ruleByPath(r *http.Request, st *model.State) (*model.BlockRule, error) {
+	id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64)
+	rule := st.BlockRules[id]
+	if rule == nil {
+		return nil, notFound("rule %s not found", r.PathValue("id"))
+	}
+	return rule, nil
+}
+
+func (s *Server) createBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	body, err := readBody(r)
+	if err != nil {
+		return err
+	}
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleCreate, 0, body)}, ruleObject)
+}
+
+func (s *Server) updateBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	rule, err := s.ruleByPath(r, st)
+	if err != nil {
+		return err
+	}
+	body, err := readBody(r)
+	if err != nil {
+		return err
+	}
+	id := rule.ID
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleUpdate, id, body)}, func(st *model.State, _ int64) any { return ruleObject(st, id) })
+}
+
+func (s *Server) deleteBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	rule, err := s.ruleByPath(r, st)
+	if err != nil {
+		return err
+	}
+	return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleDelete, rule.ID, nil)}, nil)
+}
+
+// deleteBlockRulesByName: DELETE /blocking/rules?name=NAME[&view=VIEW]
+// deletes the rules for a domain (in one view, or all).
+func (s *Server) deleteBlockRulesByName(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	st, err := s.state(r)
+	if err != nil {
+		return err
+	}
+	q := r.URL.Query()
+	name := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(q.Get("name"))), ".")
+	if name == "" {
+		return badRequest("give the domain of the rules to delete (name=)")
+	}
+	var viewID int64 = -1
+	if vn := q.Get("view"); vn != "" {
+		v := st.ViewByName(vn)
+		if v == nil {
+			return notFound("view %s not found", vn)
+		}
+		viewID = v.ID
+	}
+	var reqs []engine.Request
+	for _, rule := range st.SortedBlockRules() {
+		if rule.Name != name {
+			continue
+		}
+		if viewID >= 0 && !slices.Contains(st.BlockingViews(rule.ViewIDs), viewID) {
+			continue
+		}
+		reqs = append(reqs, request(engine.OpBlockRuleDelete, rule.ID, nil))
+	}
+	if len(reqs) == 0 {
+		if queryBool(r, "missing_ok") {
+			writeJSON(w, http.StatusOK, api.MutationResponse{})
+			return nil
+		}
+		return notFound("there is no rule for %s", name)
+	}
+	return s.mutate(w, r, p, "", reqs, nil)
+}
+
+// checkBlocking: GET /blocking/check?name=NAME says what blocking does
+// with a name in each view.
+func (s *Server) checkBlocking(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
+	if err := requireBlocking(p); err != nil {
+		return err
+	}
+	st, err := s.store.LoadState(r.Context())
+	if err != nil {
+		return err
+	}
+	out, err := s.eng.CheckBlocking(st, r.URL.Query().Get("name"))
+	if err != nil {
+		return err
+	}
+	if out == nil {
+		out = []blocking.Match{}
+	}
+	writeJSON(w, http.StatusOK, out)
+	return nil
+}
+
+// blocklistErrors lists the enabled blocklists whose last update failed,
+// for the dashboard.
+func (s *Server) blocklistErrors(ctx context.Context) []api.BlocklistError {
+	st, err := s.store.LoadState(ctx)
+	if err != nil || len(st.Blocklists) == 0 {
+		return nil
+	}
+	statuses, err := s.store.BlocklistStatuses(ctx)
+	if err != nil {
+		return nil
+	}
+	var out []api.BlocklistError
+	for _, b := range st.SortedBlocklists() {
+		if s := statuses[b.ID]; s != nil && s.Error != "" && !b.Disabled {
+			out = append(out, api.BlocklistError{Name: b.Name, Error: s.Error, UpdatedAt: s.UpdatedAt})
+		}
+	}
+	return out
+}

internal/server/h_changes.go

@@ -279,6 +279,9 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa
 		resp.Backup = s.backupStatus(r.Context(), now)
 		resp.TOTP = s.totpOverview(r.Context())
 	}
+	if p.Can(auth.Action{Resource: auth.ResBlocking}) {
+		resp.BlocklistErrors = s.blocklistErrors(r.Context())
+	}
 	writeJSON(w, http.StatusOK, resp)
 	return nil
 }

internal/server/routes.go

@@ -56,6 +56,19 @@ func (s *Server) routes() {
 	m.HandleFunc("DELETE "+p+"/views/{id}", s.authed(s.deleteView))
 	m.HandleFunc("PUT "+p+"/views/{id}/color", s.authed(s.setViewColor))
 	// ACLs
+	// Blocking
+	m.HandleFunc("GET "+p+"/blocking/lists", s.authed(s.listBlocklists))
+	m.HandleFunc("POST "+p+"/blocking/lists", s.authed(s.createBlocklist))
+	m.HandleFunc("GET "+p+"/blocking/lists/{id}", s.authed(s.getBlocklist))
+	m.HandleFunc("PATCH "+p+"/blocking/lists/{id}", s.authed(s.updateBlocklist))
+	m.HandleFunc("DELETE "+p+"/blocking/lists/{id}", s.authed(s.deleteBlocklist))
+	m.HandleFunc("POST "+p+"/blocking/lists/{id}/refresh", s.authed(s.refreshBlocklist))
+	m.HandleFunc("GET "+p+"/blocking/rules", s.authed(s.listBlockRules))
+	m.HandleFunc("POST "+p+"/blocking/rules", s.authed(s.createBlockRule))
+	m.HandleFunc("DELETE "+p+"/blocking/rules", s.authed(s.deleteBlockRulesByName))
+	m.HandleFunc("PATCH "+p+"/blocking/rules/{id}", s.authed(s.updateBlockRule))
+	m.HandleFunc("DELETE "+p+"/blocking/rules/{id}", s.authed(s.deleteBlockRule))
+	m.HandleFunc("GET "+p+"/blocking/check", s.authed(s.checkBlocking))
 	m.HandleFunc("GET "+p+"/acls", s.authed(s.listACLs))
 	m.HandleFunc("POST "+p+"/acls", s.authed(s.createACL))
 	m.HandleFunc("GET "+p+"/acls/{id}", s.authed(s.getACL))