Add blocking: blocklists and block/allow rules with BIND response policy zones

jarld downloads blocklists (Adblock, hosts or plain domains), checks them
daily and reloads named only when a list changed; your own block and
allow rules take precedence, and the most specific rule wins. Lists and
rules apply per view (default: every view that offers recursion). Web UI
Blocking page with name checks, jarl blocklist and jarl block, the
/blocking API, blocked counts in the statistics. Database schema 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/stats/blocking_test.go

@@ -0,0 +1,111 @@
+package stats
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestParseRewrite(t *testing.T) {
+	loc := time.UTC
+	// Real lines from named 9.20.24.
+	rw, ok := ParseRewrite(`09-Oct-2026 23:49:13.089 client @0x7bd607320000 127.0.0.1#56905 (x.ads.example): view internal: rpz QNAME NXDOMAIN rewrite x.ads.example/A/IN via x.ads.example.rules.jarl.invalid`, loc)
+	if !ok || rw.Client.String() != "127.0.0.1" || rw.Port != "56905" || rw.View != "internal" || rw.Name != "x.ads.example" || rw.Action != "NXDOMAIN" || policySource(rw.Via) != "rules" {
+		t.Errorf("rules rewrite: %+v (%s)", rw, policySource(rw.Via))
+	}
+	rw, ok = ParseRewrite(`09-Oct-2026 23:49:13.378 client @0x7bd607320000 10.0.0.9#57347 (Tracker.Example): rpz QNAME NXDOMAIN rewrite tracker.example/AAAA/IN via tracker.example.hagezi-pro.blocklist.jarl.invalid`, loc)
+	if !ok || rw.View != "" || rw.Name != "tracker.example" || policySource(rw.Via) != "hagezi-pro" {
+		t.Errorf("list rewrite: %+v (%s)", rw, policySource(rw.Via))
+	}
+	if policySource("ads.example.my-own-rpz.") != "ads.example.my-own-rpz" {
+		t.Error("a policy zone that isn't Jarl's")
+	}
+	if _, ok := ParseRewrite(`09-Oct-2026 23:49:13.378 client @0x7bd607320000 10.0.0.9#57347 (x): view internal: response: x IN A NOERROR 1 0 1 + (10.0.0.2)`, loc); ok {
+		t.Error("a response line parsed as a rewrite")
+	}
+}
+
+func TestCollectorBlocking(t *testing.T) {
+	ctx := context.Background()
+	loc := time.UTC
+	dir := t.TempDir()
+	logFile := filepath.Join(dir, "responses.log")
+	store, err := OpenStore(filepath.Join(dir, "stats.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+	named := newFakeNamed(t)
+	clk := &clock{time.Date(2026, 10, 5, 10, 0, 30, 0, loc)}
+	opts := Options{QueryLog: true, LogFile: logFile, StatisticsChannel: "unused", TopN: 5,
+		RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour,
+		Exclude: []netip.Prefix{netip.MustParsePrefix("10.0.0.250/32")}, Location: loc}
+	c := NewCollector(store, opts, func() map[string][]string { return nil }, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	c.poller.URL = named.srv.URL
+	c.now = clk.now
+	c.start(ctx)
+
+	at := time.Date(2026, 10, 5, 10, 0, 40, 0, loc).Format(logTimeLayout)
+	rewrite := func(client, port, name, action, via string) string {
+		return fmt.Sprintf("%s client @0x7f0000000000 %s#%s (%s): view internal: rpz QNAME %s rewrite %s/A/IN via %s\n", at, client, port, name, action, name, via)
+	}
+	response := func(client, port, name, rcode string) string {
+		return fmt.Sprintf("%s client @0x7f0000000000 %s#%s (%s): view internal: response: %s IN A %s 0 0 2 +E(0)K (10.0.0.2)\n", at, client, port, name, name, rcode)
+	}
+	f, _ := os.Create(logFile)
+	for _, l := range []string{
+		rewrite("10.0.0.5", "1001", "ads.example", "NXDOMAIN", "ads.example.hagezi.blocklist.jarl.invalid"),
+		response("10.0.0.5", "1001", "ads.example", "NXDOMAIN"),
+		rewrite("10.0.0.5", "1002", "x.ads.example", "NXDOMAIN", "x.ads.example.hagezi.blocklist.jarl.invalid"),
+		response("10.0.0.5", "1002", "x.ads.example", "NXDOMAIN"),
+		rewrite("10.0.0.6", "1003", "ads.example", "NXDOMAIN", "ads.example.rules.jarl.invalid"),
+		response("10.0.0.6", "1003", "ads.example", "NXDOMAIN"),
+		// An allow rule: answered normally, not blocked.
+		rewrite("10.0.0.6", "1004", "cdn.ads.example", "PASSTHRU", "cdn.ads.example.rules.jarl.invalid"),
+		response("10.0.0.6", "1004", "cdn.ads.example", "NOERROR"),
+		// A real NXDOMAIN.
+		response("10.0.0.6", "1005", "typo.example", "NXDOMAIN"),
+		// An excluded client.
+		rewrite("10.0.0.250", "1006", "ads.example", "NXDOMAIN", "ads.example.hagezi.blocklist.jarl.invalid"),
+		response("10.0.0.250", "1006", "ads.example", "NXDOMAIN"),
+	} {
+		f.WriteString(l)
+	}
+	f.Close()
+	c.readLog()
+	named.add(6, 4)
+	clk.t = time.Date(2026, 10, 5, 10, 1, 2, 0, loc)
+	c.flush(ctx, false)
+
+	m := time.Date(2026, 10, 5, 10, 0, 0, 0, loc).Unix()
+	got, _ := store.sums(ctx, ResMinute, m, m+60, "")
+	for k, v := range map[string]int64{"blocked": 3, "blocklist:hagezi": 2, "blocklist:rules": 1, "responses": 5} {
+		if got[k] != v {
+			t.Errorf("%s = %d, want %d (all: %v)", k, got[k], v, got)
+		}
+	}
+	if b, _ := store.TopsAt(ctx, ResHour, m, KindBlocked); len(b) != 2 || b[0] != (Entry{Key: "ads.example", Count: 2}) {
+		t.Errorf("blocked names = %v", b)
+	}
+	if nx, _ := store.TopsAt(ctx, ResHour, m, KindNXDomain); len(nx) != 1 || nx[0].Key != "typo.example" {
+		t.Errorf("NXDOMAIN names should leave out blocked ones: %v", nx)
+	}
+	q := &Querier{Store: store, Opts: opts, Now: clk.now}
+	sum, err := q.Summary(ctx, time.Hour, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 6 queries, 4 NXDOMAIN of which 3 blocked.
+	if sum.Blocked != 3 || sum.BlockedRatio == nil || *sum.BlockedRatio != 0.5 || sum.NXDomainRatio == nil || *sum.NXDomainRatio != 1.0/6 {
+		t.Errorf("summary: blocked %d ratio %v nxdomain %v", sum.Blocked, sum.BlockedRatio, sum.NXDomainRatio)
+	}
+	if len(sum.Blocklists) != 2 || sum.Blocklists[0] != (Entry{Key: "hagezi", Count: 2}) || len(sum.Tops[KindBlocked]) != 2 {
+		t.Errorf("summary lists %v, tops %v", sum.Blocklists, sum.Tops[KindBlocked])
+	}
+}

internal/stats/collector.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"jarl/internal/blocking"
 )
 
 // Kinds of top lists.
@@ -21,10 +23,11 @@ const (
 	KindNXDomain = "nxdomain" // names answered with NXDOMAIN
 	KindServFail = "servfail" // names answered with SERVFAIL
 	KindRefused  = "refused"  // clients that were refused
+	KindBlocked  = "blocked"  // names blocked (docs/blocking.md)
 )
 
 // Kinds lists the top-list kinds.
-var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused}
+var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused, KindBlocked}
 
 // OtherZone is the zone key for names outside every local zone (answered
 // by recursion, or refused).
@@ -99,6 +102,9 @@ type Collector struct {
 	recentL  []int64 // response log lines per minute, last 10 minutes
 	linesMin int64
 	ensureAt time.Time
+	// rewritten holds queries named blocked (client#port|name) until their
+	// response line arrives.
+	rewritten map[string]bool
 }
 
 // maxKeys bounds the per-hour maps; rarely seen keys are dropped first.
@@ -296,8 +302,15 @@ func (c *Collector) HandleLine(line string) {
 func (c *Collector) handleLine(line string) {
 	ev, ok := ParseResponse(line, c.opts.Location)
 	if !ok {
+		if rw, ok := ParseRewrite(line, c.opts.Location); ok {
+			c.handleRewrite(rw)
+		}
 		return
 	}
+	blocked := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
+	if blocked {
+		delete(c.rewritten, ev.Client.String()+"#"+ev.Port+"|"+ev.Name)
+	}
 	c.status.LinesRead++
 	c.linesMin++
 	t := ev.Time
@@ -324,12 +337,14 @@ func (c *Collector) handleLine(line string) {
 	tops := hours[hour]
 	tops[KindName][ev.Name]++
 	tops[KindClient][client]++
-	switch ev.RCode {
-	case "NXDOMAIN":
+	switch {
+	case blocked:
+		// Counted by handleRewrite; not a name that doesn't exist.
+	case ev.RCode == "NXDOMAIN":
 		tops[KindNXDomain][ev.Name]++
-	case "SERVFAIL":
+	case ev.RCode == "SERVFAIL":
 		tops[KindServFail][ev.Name]++
-	case "REFUSED":
+	case ev.RCode == "REFUSED":
 		tops[KindRefused][client]++
 	}
 	for _, m := range tops {
@@ -339,6 +354,48 @@ func (c *Collector) handleLine(line string) {
 	}
 }
 
+// handleRewrite counts a query that blocking answered: "blocked", and
+// "blocklist:NAME" for the list (or "rules") that matched.
+func (c *Collector) handleRewrite(rw Rewrite) {
+	if rw.Action == "PASSTHRU" {
+		return // an allow rule: answered normally
+	}
+	for _, p := range c.opts.Exclude {
+		if p.Contains(rw.Client) {
+			return
+		}
+	}
+	if c.rewritten == nil || len(c.rewritten) > 10_000 {
+		// Rewrites whose response line never came (named only logs
+		// responses while response logging is on) don't pile up.
+		c.rewritten = map[string]bool{}
+	}
+	c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true
+	minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time)
+	c.count(minute, "blocked", 1)
+	c.count(minute, "blocklist:"+policySource(rw.Via), 1)
+	hours := c.hours
+	if hour < c.liveHour {
+		hours = c.late
+	}
+	hours[hour] = c.hourMaps(hours, hour)
+	hours[hour][KindBlocked][rw.Name]++
+}
+
+// policySource names what a rewrite came from: a blocklist's name, "rules"
+// for the rules entered in Jarl, or the policy zone for others.
+func policySource(via string) string {
+	via = strings.TrimSuffix(via, ".")
+	if strings.HasSuffix(via, "."+blocking.RulesZone) {
+		return "rules"
+	}
+	if i := strings.LastIndex(via, ".blocklist.jarl.invalid"); i > 0 && i+len(".blocklist.jarl.invalid") == len(via) {
+		rest := via[:i]
+		return rest[strings.LastIndex(rest, ".")+1:]
+	}
+	return via
+}
+
 // shrink drops the rarest keys until the map is well under maxKeys.
 func shrink(m map[string]int64) {
 	for limit := int64(1); len(m) > maxKeys*8/10; limit++ {

internal/stats/parse.go

@@ -11,18 +11,36 @@ import (
 type Event struct {
 	Time   time.Time
 	Client netip.Addr
+	Port   string // the client's port: with Client and Name, it ties an RPZ rewrite to its response
 	View   string // "" without views
 	Name   string // lowercase, without the final dot ("." for the root)
 	Type   string // e.g. A, AAAA, PTR
 	RCode  string // e.g. NOERROR, NXDOMAIN
 }
 
+// Rewrite is a response policy (RPZ) rewrite named logged: blocking.
+type Rewrite struct {
+	Time   time.Time
+	Client netip.Addr
+	Port   string
+	View   string
+	Name   string // the query name, normalized like Event.Name
+	Action string // NXDOMAIN, NODATA, PASSTHRU, DROP, ...
+	Via    string // the policy zone entry: OWNER.ZONE
+}
+
 // responseRe matches named's response log lines (BIND 9.20, print-time):
 //
 //	07-Oct-2026 19:46:22.195 client @0x7135... 10.10.0.37#51709 (nas.adbraun.lol): view internal: response: nas.adbraun.lol IN A NOERROR 1 0 1 +E(0)K (10.10.0.2)
 //
 // The "view NAME: " part is missing when no views are configured.
-var responseRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#\d+ \(.*?\): (?:view (.+?): )?response: (\S+) (\S+) (\S+) ([A-Z0-9]+) `)
+var responseRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#(\d+) \(.*?\): (?:view (.+?): )?response: (\S+) (\S+) (\S+) ([A-Z0-9]+) `)
+
+// rewriteRe matches named's RPZ rewrite lines (category rpz), logged just
+// before the response line of the same query:
+//
+//	09-Oct-2026 23:49:13.698 client @0x7135... 10.10.0.37#54468 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid
+var rewriteRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#(\d+) \((.*?)\): (?:view (.+?): )?rpz \S+ (\S+) rewrite \S+ via (\S+)`)
 
 const logTimeLayout = "02-Jan-2006 15:04:05.000"
 
@@ -44,13 +62,31 @@ func ParseResponse(line string, loc *time.Location) (ev Event, ok bool) {
 	return Event{
 		Time:   t,
 		Client: client.Unmap(),
-		View:   m[3],
-		Name:   normalizeName(m[4]),
-		Type:   m[6],
-		RCode:  m[7],
+		Port:   m[3],
+		View:   m[4],
+		Name:   normalizeName(m[5]),
+		Type:   m[7],
+		RCode:  m[8],
 	}, true
 }
 
+// ParseRewrite parses an RPZ rewrite log line.
+func ParseRewrite(line string, loc *time.Location) (rw Rewrite, ok bool) {
+	m := rewriteRe.FindStringSubmatch(line)
+	if m == nil {
+		return rw, false
+	}
+	t, err := time.ParseInLocation(logTimeLayout, m[1], loc)
+	if err != nil {
+		return rw, false
+	}
+	client, err := netip.ParseAddr(m[2])
+	if err != nil {
+		return rw, false
+	}
+	return Rewrite{Time: t, Client: client.Unmap(), Port: m[3], Name: normalizeName(m[4]), View: m[5], Action: m[6], Via: strings.ToLower(m[7])}, true
+}
+
 func normalizeName(n string) string {
 	n = strings.ToLower(n)
 	if n == "." {

internal/stats/query.go

@@ -153,14 +153,19 @@ type Summary struct {
 		Queries int64     `json:"queries"`
 		Per     string    `json:"per"` // minute, hour or day
 	} `json:"peak,omitempty"`
-	RCodes        []Entry            `json:"rcodes"`
-	QTypes        []Entry            `json:"qtypes"`
-	Views         []Entry            `json:"views"`
-	Zones         []Entry            `json:"zones"`
-	Answers       []Entry            `json:"answers"`   // authoritative, recursive
-	Transport     []Entry            `json:"transport"` // udp, tcp
-	CacheHitRatio *float64           `json:"cache_hit_ratio,omitempty"`
+	RCodes        []Entry  `json:"rcodes"`
+	QTypes        []Entry  `json:"qtypes"`
+	Views         []Entry  `json:"views"`
+	Zones         []Entry  `json:"zones"`
+	Answers       []Entry  `json:"answers"`   // authoritative, recursive
+	Transport     []Entry  `json:"transport"` // udp, tcp
+	CacheHitRatio *float64 `json:"cache_hit_ratio,omitempty"`
+	// NXDomainRatio leaves out answers blocking gave (which are
+	// NXDOMAIN too); Blocked counts those (docs/blocking.md).
 	NXDomainRatio *float64           `json:"nxdomain_ratio,omitempty"`
+	Blocked       int64              `json:"blocked"`
+	BlockedRatio  *float64           `json:"blocked_ratio,omitempty"`
+	Blocklists    []Entry            `json:"blocklists,omitempty"`
 	Tops          map[string][]Entry `json:"tops,omitempty"`
 }
 
@@ -181,9 +186,14 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int
 		r := float64(h) / float64(h+m)
 		s.CacheHitRatio = &r
 	}
+	s.Blocked, s.Blocklists = all["blocked"], sorted(all, "blocklist:")
 	if s.Queries > 0 {
-		r := float64(all["rcode:NXDOMAIN"]) / float64(s.Queries)
+		r := float64(max(all["rcode:NXDOMAIN"]-s.Blocked, 0)) / float64(s.Queries)
 		s.NXDomainRatio = &r
+		if s.Blocked > 0 {
+			b := float64(s.Blocked) / float64(s.Queries)
+			s.BlockedRatio = &b
+		}
 	}
 	per, err := q.Store.series(ctx, res, from, to, "queries")
 	if err != nil {
@@ -209,7 +219,7 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int
 	}
 	if topLimit > 0 {
 		s.Tops = map[string][]Entry{}
-		for _, k := range []string{KindName, KindClient, KindNXDomain} {
+		for _, k := range []string{KindName, KindClient, KindNXDomain, KindBlocked} {
 			es, err := q.Top(ctx, k, since, topLimit)
 			if err != nil {
 				return nil, err
@@ -246,7 +256,7 @@ func (q *Querier) Breakdown(ctx context.Context, group string, since time.Durati
 }
 
 // Groups are the counter groups Breakdown accepts.
-var Groups = []string{"qtype", "rcode", "view", "zone", "answer", "transport", "rtt", "dnssec", "cache", "ratelimit"}
+var Groups = []string{"qtype", "rcode", "view", "zone", "answer", "transport", "rtt", "dnssec", "cache", "ratelimit", "blocklist"}
 
 // Timeline is a set of series over time.
 type Timeline struct {