Add blocking: blocklists and block/allow rules with BIND response policy zones

jarld downloads blocklists (Adblock, hosts or plain domains), checks them
daily and reloads named only when a list changed; your own block and
allow rules take precedence, and the most specific rule wins. Lists and
rules apply per view (default: every view that offers recursion). Web UI
Blocking page with name checks, jarl blocklist and jarl block, the
/blocking API, blocked counts in the statistics. Database schema 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/jarl/blocking.go

@@ -0,0 +1,400 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"jarl/internal/api"
+	"jarl/internal/blocking"
+)
+
+// ---- blocking (docs/blocking.md)
+
+// viewsBody adds the --view flags to a request body: names, or "auto" for
+// every view that offers recursion.
+func viewsBody(body map[string]any, views []string) {
+	if len(views) == 1 && strings.EqualFold(views[0], "auto") {
+		body["views"] = []string{}
+	} else if len(views) > 0 {
+		body["views"] = views
+	}
+}
+
+func viewsText(views []string, auto bool) string {
+	switch {
+	case auto:
+		return "auto (" + strings.Join(views, ", ") + ")"
+	case len(views) == 0:
+		return "-"
+	}
+	return strings.Join(views, ", ")
+}
+
+func blocklistCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "blocklist", Aliases: []string{"blocklists"}, Short: "Blocklists: domains to block, downloaded and kept up to date",
+		Long: `Blocklists are downloaded by jarld and turned into response policy zones, so
+named answers "doesn't exist" for the domains on them (docs/blocking.md).
+Hosts, plain domain and Adblock (||domain^) lists work. Your own block and
+allow rules ('jarl block') take precedence over every list.`}
+	var views []string
+	var comment string
+
+	list := &cobra.Command{
+		Use:   "list",
+		Short: "List blocklists and their state",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var lists []api.Blocklist
+			if err := c.Get(ctx(), "/blocking/lists", nil, &lists); err != nil {
+				return err
+			}
+			return output(lists, func(w io.Writer) {
+				fmt.Fprintln(w, "NAME\tSTATE\tDOMAINS\tVIEWS\tUPDATED\tMEMORY\tURL")
+				for _, b := range lists {
+					domains, updated, mem := "-", "-", "-"
+					if s := b.Status; s != nil && s.UpdatedAt != nil {
+						domains, updated = fmt.Sprint(s.Entries), s.UpdatedAt.Local().Format("2006-01-02 15:04")
+					}
+					if b.MemoryMB > 0 {
+						mem = fmt.Sprintf("~%d MB", b.MemoryMB)
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", b.Name, b.State, domains, viewsText(b.Views, b.AutoViews), updated, mem, b.URL)
+				}
+				for _, b := range lists {
+					if b.Status != nil && b.Status.Error != "" {
+						fmt.Fprintf(w, "\n%s: %s\n", b.Name, b.Status.Error)
+					}
+				}
+			})
+		},
+	}
+
+	add := &cobra.Command{
+		Use:   "add NAME URL",
+		Short: "Add a blocklist (jarld downloads it right away)",
+		Long: `Add a blocklist. NAME is a short label (letters, digits, hyphens). Without
+--view the list applies to every view that offers recursion.
+
+  jarl blocklist add hagezi-pro https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt
+  jarl blocklist add stevenblack https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts --view internal`,
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			body := map[string]any{"name": args[0], "url": args[1]}
+			viewsBody(body, views)
+			if comment != "" {
+				body["comment"] = comment
+			}
+			resp, err := c.Mutate(ctx(), "POST", "/blocking/lists", nil, body)
+			if err != nil {
+				return err
+			}
+			if err := report(resp); err != nil || flagQuiet || !resp.Applied {
+				return err
+			}
+			fmt.Println("jarld downloads the list in the background; 'jarl blocklist list' shows when it's in use.")
+			return nil
+		},
+	}
+	add.Flags().StringArrayVar(&views, "view", nil, "a view it applies to (repeat for several; default: every view that offers recursion)")
+	add.Flags().StringVar(&comment, "comment", "", "comment")
+
+	var rename, newURL string
+	update := &cobra.Command{
+		Use:   "update LIST",
+		Short: "Change a blocklist's name, URL, views or comment",
+		Example: `  jarl blocklist update hagezi --url https://example.org/new-list.txt
+  jarl blocklist update hagezi --view internal --view guest
+  jarl blocklist update hagezi --view auto     # every view that offers recursion`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			body := map[string]any{}
+			f := cmd.Flags()
+			if f.Changed("name") {
+				body["name"] = rename
+			}
+			if f.Changed("url") {
+				body["url"] = newURL
+			}
+			if f.Changed("comment") {
+				body["comment"] = comment
+			}
+			viewsBody(body, views)
+			if len(body) == 0 {
+				return fmt.Errorf("nothing to change (see --help)")
+			}
+			resp, err := c.Mutate(ctx(), "PATCH", "/blocking/lists/"+url.PathEscape(args[0]), nil, body)
+			if err != nil {
+				return err
+			}
+			return report(resp)
+		},
+	}
+	update.Flags().StringVar(&rename, "name", "", "new name")
+	update.Flags().StringVar(&newURL, "url", "", "new URL")
+	update.Flags().StringArrayVar(&views, "view", nil, `a view it applies to (repeat for several; "auto": every view that offers recursion)`)
+	update.Flags().StringVar(&comment, "comment", "", "new comment")
+
+	toggle := func(disable bool) *cobra.Command {
+		verb := map[bool]string{true: "disable", false: "enable"}[disable]
+		return &cobra.Command{
+			Use:   verb + " LIST",
+			Short: map[bool]string{true: "Stop using a blocklist (it is kept, and re-enabled without downloading)", false: "Use a disabled blocklist again"}[disable],
+			Args:  cobra.ExactArgs(1),
+			RunE: func(cmd *cobra.Command, args []string) error {
+				c, err := newClient()
+				if err != nil {
+					return err
+				}
+				resp, err := c.Mutate(ctx(), "PATCH", "/blocking/lists/"+url.PathEscape(args[0]), nil, map[string]any{"disabled": disable})
+				if err != nil {
+					return err
+				}
+				return report(resp)
+			},
+		}
+	}
+
+	var yes bool
+	del := &cobra.Command{
+		Use:   "delete LIST",
+		Short: "Delete a blocklist",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !confirm("Delete blocklist "+args[0]+"?", yes) {
+				return fmt.Errorf("canceled")
+			}
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			resp, err := c.Mutate(ctx(), "DELETE", "/blocking/lists/"+url.PathEscape(args[0]), nil, nil)
+			if err != nil {
+				return err
+			}
+			return report(resp)
+		},
+	}
+	del.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask")
+
+	var force bool
+	refresh := &cobra.Command{
+		Use:   "refresh [LIST...]",
+		Short: "Download blocklists now (all enabled ones without LIST)",
+		Long: `Download blocklists now instead of waiting for their next check
+(blocking.refresh in jarl.yaml, daily by default). A list is only replaced
+when its content changed; --force downloads it even when the server says
+it hasn't.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			names := args
+			if len(names) == 0 {
+				var lists []api.Blocklist
+				if err := c.Get(ctx(), "/blocking/lists", nil, &lists); err != nil {
+					return err
+				}
+				for _, b := range lists {
+					if b.State != "disabled" && b.State != "idle" {
+						names = append(names, b.Name)
+					}
+				}
+			}
+			if flagDryRun {
+				fmt.Printf("would download: %s\n", strings.Join(names, ", "))
+				return nil
+			}
+			failed := 0
+			for _, n := range names {
+				var b api.Blocklist
+				start := time.Now()
+				err := c.Do(ctx(), "POST", "/blocking/lists/"+url.PathEscape(n)+"/refresh", q("force", map[bool]string{true: "1"}[force]), nil, &b)
+				switch {
+				case err != nil:
+					failed++
+					fmt.Printf("%s: %v\n", n, err)
+				case b.Status != nil && b.Status.Error != "":
+					failed++
+					fmt.Printf("%s: %s (the previous version stays in use)\n", n, b.Status.Error)
+				case b.Status != nil:
+					fmt.Printf("%s: %d domains (%d lines skipped), checked in %s\n", n, b.Status.Entries, b.Status.Skipped, time.Since(start).Round(100*time.Millisecond))
+				}
+			}
+			if failed > 0 {
+				return &checkFailed{fmt.Sprintf("%d of %d lists could not be updated", failed, len(names))}
+			}
+			return nil
+		},
+	}
+	refresh.Flags().BoolVar(&force, "force", false, "download even when the server says the list hasn't changed")
+
+	cmd.AddCommand(list, add, update, toggle(true), toggle(false), del, refresh)
+	return cmd
+}
+
+func blockCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "block", Short: "Your own block and allow rules, and checking what is blocked",
+		Long: `Rules block a domain, or allow it despite the blocklists ('jarl blocklist').
+A rule covers the domain and every name below it, unless --exact. Rules
+take precedence over every list, and the most specific rule wins: allow
+cdn.example.com overrides block example.com for cdn.example.com and the
+names below it (docs/blocking.md).`}
+	var views []string
+	var comment, view string
+	var exact bool
+
+	addRule := func(action string) *cobra.Command {
+		c := &cobra.Command{
+			Use:   map[string]string{"block": "add DOMAIN", "allow": "allow DOMAIN"}[action],
+			Short: map[string]string{"block": "Block a domain (and the names below it)", "allow": "Allow a domain (and the names below it) that a blocklist blocks"}[action],
+			Example: map[string]string{
+				"block": "  jarl block add doubleclick.net\n  jarl block add telemetry.example.com --exact --view internal",
+				"allow": "  jarl block allow cdn.example.com\n  jarl block allow login.example.com --exact",
+			}[action],
+			Args: cobra.ExactArgs(1),
+			RunE: func(cmd *cobra.Command, args []string) error {
+				c, err := newClient()
+				if err != nil {
+					return err
+				}
+				body := map[string]any{"name": args[0], "action": action}
+				if exact {
+					body["exact"] = true
+				}
+				viewsBody(body, views)
+				if comment != "" {
+					body["comment"] = comment
+				}
+				resp, err := c.Mutate(ctx(), "POST", "/blocking/rules", nil, body)
+				if err != nil {
+					return err
+				}
+				return report(resp)
+			},
+		}
+		c.Flags().BoolVar(&exact, "exact", false, "only this name, not the names below it")
+		c.Flags().StringArrayVar(&views, "view", nil, "a view it applies to (repeat for several; default: every view that offers recursion)")
+		c.Flags().StringVar(&comment, "comment", "", "comment")
+		return c
+	}
+
+	list := &cobra.Command{
+		Use:   "list",
+		Short: "List your block and allow rules",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var rules []api.BlockRule
+			if err := c.Get(ctx(), "/blocking/rules", nil, &rules); err != nil {
+				return err
+			}
+			return output(rules, func(w io.Writer) {
+				fmt.Fprintln(w, "ID\tACTION\tDOMAIN\tCOVERS\tVIEWS\tCOMMENT")
+				for _, r := range rules {
+					covers := "and below"
+					if r.Exact {
+						covers = "exact"
+					}
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\n", r.ID, r.Action, r.Name, covers, viewsText(r.Views, r.AutoViews), r.Comment)
+				}
+			})
+		},
+	}
+
+	var id int64
+	var missingOK bool
+	del := &cobra.Command{
+		Use:   "delete DOMAIN  |  delete --id ID",
+		Short: "Delete the rules for a domain (or one rule by ID)",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var resp *api.MutationResponse
+			switch {
+			case id > 0:
+				resp, err = c.Mutate(ctx(), "DELETE", fmt.Sprintf("/blocking/rules/%d", id), nil, nil)
+			case len(args) == 1:
+				qv := q("name", args[0], "view", view)
+				if missingOK {
+					qv.Set("missing_ok", "1")
+				}
+				resp, err = c.Mutate(ctx(), "DELETE", "/blocking/rules", qv, nil)
+			default:
+				return fmt.Errorf("give DOMAIN, or --id")
+			}
+			if err != nil {
+				return err
+			}
+			if !resp.Applied && !resp.Staged && !resp.DryRun {
+				fmt.Println("nothing to do")
+				return nil
+			}
+			return report(resp)
+		},
+	}
+	del.Flags().Int64Var(&id, "id", 0, "delete the rule with this ID")
+	del.Flags().StringVar(&view, "view", "", "only the rules that apply in this view")
+	del.Flags().BoolVar(&missingOK, "missing-ok", false, "succeed when there is no such rule")
+
+	check := &cobra.Command{
+		Use:   "check DOMAIN",
+		Short: "Show whether a name is blocked, and by which rule or list",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			var matches []blocking.Match
+			if err := c.Get(ctx(), "/blocking/check", q("name", args[0]), &matches); err != nil {
+				return err
+			}
+			return output(matches, func(w io.Writer) {
+				if len(matches) == 0 {
+					fmt.Fprintln(w, "Blocking isn't set up in any view.")
+					return
+				}
+				for _, m := range matches {
+					where := ""
+					if m.View != "" {
+						where = "view " + m.View + ":\t"
+					}
+					switch {
+					case m.Result == "none":
+						fmt.Fprintf(w, "%snot blocked\n", where)
+					case m.List != "":
+						fmt.Fprintf(w, "%sblocked by blocklist %s (entry %s)\n", where, m.List, m.Entry)
+					default:
+						fmt.Fprintf(w, "%s%s by your rule %d (entry %s)\n", where, m.Result, m.RuleID, m.Entry)
+					}
+				}
+			})
+		},
+	}
+
+	cmd.AddCommand(addRule("block"), addRule("allow"), list, del, check)
+	return cmd
+}

cmd/jarl/complete.go

@@ -76,6 +76,8 @@ func argCompletion(c *cobra.Command) cobra.CompletionFunc {
 			return append([]string{"default"}, model.ViewColors...), cobra.ShellCompDirectiveNoFileComp
 		case p == "ACL":
 			return lookup(cmd, "acl")
+		case p == "LIST":
+			return lookup(cmd, "blocklist")
 		case p == "USER":
 			return lookup(cmd, "user")
 		case p == "KEY" && parent == "tsig":
@@ -199,6 +201,14 @@ func lookup(cmd *cobra.Command, kind string) ([]cobra.Completion, cobra.ShellCom
 		for _, v := range views {
 			addName(v.Name, v.Comment)
 		}
+	case "blocklist":
+		var lists []api.Blocklist
+		if c.Get(ctx(), "/blocking/lists", nil, &lists) != nil {
+			return nil, none
+		}
+		for _, b := range lists {
+			addName(b.Name, b.URL)
+		}
 	case "acl":
 		var acls []api.ACL
 		if c.Get(ctx(), "/acls", nil, &acls) != nil {

cmd/jarl/main.go

@@ -250,7 +250,7 @@ Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found,
 			root.AddCommand(c)
 		}
 	}
-	add("dns", zoneCmd(), recordCmd(), viewCmd(), aclCmd(), tsigCmd(), optionCmd(), settingsCmd(), acmeCmd(), searchCmd(), statsCmd())
+	add("dns", zoneCmd(), recordCmd(), viewCmd(), aclCmd(), tsigCmd(), optionCmd(), settingsCmd(), blockCmd(), blocklistCmd(), acmeCmd(), searchCmd(), statsCmd())
 	add("changes", applyCmd(), changesetCmd(), historyCmd())
 	add("admin", statusCmd(), whoamiCmd(), userCmd(), tokenCmd(), auditCmd(), checkCmd(), resyncCmd(), configCmd())
 	setupCompletion(root)

cmd/jarl/stats.go

@@ -92,6 +92,9 @@ export the data.`,
 				if s.NXDomainRatio != nil {
 					fmt.Fprintf(w, "NXDOMAIN:\t%.1f%% of queries\n", *s.NXDomainRatio*100)
 				}
+				if s.BlockedRatio != nil {
+					fmt.Fprintf(w, "Blocked:\t%d (%.1f%% of queries): %s\n", s.Blocked, *s.BlockedRatio*100, entryLine(s.Blocklists, s.Blocked, 6))
+				}
 				fmt.Fprintf(w, "Result codes:\t%s\n", entryLine(s.RCodes, s.Queries, 6))
 				fmt.Fprintf(w, "Record types:\t%s\n", entryLine(s.QTypes, s.Queries, 8))
 				if len(s.Answers) > 0 {
@@ -103,12 +106,12 @@ export the data.`,
 				if len(s.Zones) > 0 {
 					fmt.Fprintf(w, "Zones:\t%s\n", strings.ReplaceAll(entryLine(s.Zones, 0, 10), stats.OtherZone+" ", "other "))
 				}
-				for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain} {
+				for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain, stats.KindBlocked} {
 					es := s.Tops[k]
 					if len(es) == 0 {
 						continue
 					}
-					fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names"}[k])
+					fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names", stats.KindBlocked: "Top blocked names"}[k])
 					for _, e := range es {
 						key := e.Key
 						if e.Name != "" {
@@ -125,12 +128,13 @@ export the data.`,
 
 	var limit int
 	top := &cobra.Command{
-		Use:   "top [names|clients|nxdomain|servfail|refused]",
-		Short: "The most queried names, busiest clients, names with errors, refused clients",
+		Use:   "top [names|clients|nxdomain|servfail|refused|blocked]",
+		Short: "The most queried names, busiest clients, names with errors, refused clients, blocked names",
 		Example: `  jarl stats top names --since 7d
   jarl stats top clients --limit 50
   jarl stats top nxdomain      # misconfigured hosts and typos
-  jarl stats top refused       # who is asking for what they may not have`,
+  jarl stats top refused       # who is asking for what they may not have
+  jarl stats top blocked       # what blocking stops most (docs/blocking.md)`,
 		Args: cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			kind := stats.KindName

cmd/jarld/main.go

@@ -123,6 +123,10 @@ func openApp(allowMissingConfig bool) (*app, error) {
 		NoReload:      cfg.BIND.NoReload,
 		Extras:        renderExtras(cfg),
 	}, log)
+	eng.Limits = engine.BlocklistLimits{
+		Refresh: time.Duration(cfg.Blocking.Refresh), RetryAfter: time.Hour, Timeout: 2 * time.Minute,
+		MaxBytes: int64(cfg.Blocking.MaxDownloadMB) << 20, MaxEntries: cfg.Blocking.MaxEntries,
+	}
 	return &app{cfg: cfg, log: log, store: st, eng: eng, owner: owner}, nil
 }
 
@@ -323,6 +327,7 @@ func serveCmd() *cobra.Command {
 				}
 			}()
 			go auditRetention(ctx, a)
+			go blocklistLoop(ctx, a)
 			a.log.Info("jarld starting", "version", version.String(), "database", a.cfg.Database, "bind_config", a.cfg.BIND.ConfigDir)
 			err = srv.Serve(ctx, func() { sdNotify("READY=1\nSTATUS=serving") })
 			sdNotify("STOPPING=1")
@@ -332,6 +337,28 @@ func serveCmd() *cobra.Command {
 	}
 }
 
+// blocklistLoop keeps blocklists up to date (docs/blocking.md): every few
+// minutes it downloads the lists that are due, and at once when a list
+// was added or changed.
+func blocklistLoop(ctx context.Context, a *app) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(10 * time.Second): // let jarld finish starting
+	}
+	t := time.NewTicker(5 * time.Minute)
+	defer t.Stop()
+	for {
+		a.eng.RefreshDue(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		case <-a.eng.BlocklistKicks():
+		}
+	}
+}
+
 func auditRetention(ctx context.Context, a *app) {
 	if a.cfg.Audit.RetentionDays <= 0 {
 		return