Add blocking: blocklists and block/allow rules with BIND response policy zones

jarld downloads blocklists (Adblock, hosts or plain domains), checks them
daily and reloads named only when a list changed; your own block and
allow rules take precedence, and the most specific rule wins. Lists and
rules apply per view (default: every view that offers recursion). Web UI
Blocking page with name checks, jarl blocklist and jarl block, the
/blocking API, blocked counts in the statistics. Database schema 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/web/static/assets/pages/blocking.js

@@ -0,0 +1,208 @@
+// Blocking: your block/allow rules and blocklists (docs/blocking.md).
+import { h, mount, table, input, select, checkbox, toast, errorBox, badge, confirmDialog, relTime, time } from "../ui.js";
+import { get, change, describeResult, draftQuery, can, api } from "../api.js";
+import { rerender } from "../app.js";
+import { pendingBadge } from "./zones.js";
+
+// Well-known lists in Adblock format (they say which entries cover
+// subdomains). Sizes change; these are rough guides.
+const suggestions = [
+  ["hagezi-light", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/light.txt", "Hagezi Light: small, very few false positives"],
+  ["hagezi-multi", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/multi.txt", "Hagezi Multi Normal: ads, tracking, malware; a good default"],
+  ["hagezi-pro", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt", "Hagezi Pro: more thorough, occasionally blocks something wanted"],
+  ["oisd-small", "https://small.oisd.nl/", "OISD Small: ads and tracking, very few false positives"],
+];
+
+const stateBadge = {
+  ok: ["in use", "ok"], waiting: ["downloading", "accent"], error: ["update failed", "err"],
+  disabled: ["disabled", ""], idle: ["no effect", "warn"],
+};
+
+let refreshTimer = null;
+
+export async function render(el, params, query) {
+  clearTimeout(refreshTimer);
+  const [lists, rules, views] = await Promise.all([get("/blocking/lists", draftQuery()), get("/blocking/rules", draftQuery()), get("/views")]);
+  const writable = can("blocking", true);
+  const recursive = lists.concat(rules).find((x) => x.auto_views);
+  const autoViews = recursive ? recursive.views : null;
+
+  const checkOut = h("div");
+  const checkName = input({ type: "search", placeholder: "ads.example.com", class: "mono", value: (query && query.get("check")) || "" });
+  const runCheck = async (ev) => {
+    if (ev) ev.preventDefault();
+    const name = checkName.value.trim();
+    if (!name) return;
+    try {
+      const res = await get("/blocking/check", { name });
+      mount(checkOut, checkResult(name, res, writable));
+    } catch (e) { mount(checkOut, errorBox(e)); }
+  };
+
+  mount(el,
+    h("div", { class: "page-head" },
+      h("div", { class: "grow" }, h("h1", null, "Blocking"),
+        h("div", { class: "muted" }, "Answer \"doesn't exist\" for ads, trackers and malware, for the devices that use this server to resolve names. Your rules come first, then the blocklists."))),
+    h("div", { class: "panel" },
+      h("form", { class: "actions", onsubmit: runCheck },
+        h("label", { class: "muted" }, "Check a name"), checkName, h("button", { type: "submit" }, "Check")),
+      checkOut),
+    rulesPanel(rules, views, autoViews, writable),
+    listsPanel(lists, views, autoViews, writable));
+  if (checkName.value) runCheck();
+
+  // Lists are downloaded in the background: show when they're done.
+  if (lists.some((l) => l.state === "waiting")) {
+    refreshTimer = setTimeout(() => { if (location.hash.startsWith("#/blocking") && el.isConnected) rerender(); }, 4000);
+  }
+}
+
+function checkResult(name, res, writable) {
+  if (!res.length) return h("p", { class: "muted" }, "Blocking isn't used in any view yet: add a rule or a blocklist below.");
+  const blocked = res.some((m) => m.result === "blocked");
+  return h("div", null,
+    h("ul", { class: "plain" }, res.map((m) => h("li", null,
+      m.view ? h("span", { class: "muted" }, "view " + m.view + ": ") : null,
+      m.result === "blocked" ? badge("blocked", "err") : m.result === "allowed" ? badge("allowed", "ok") : badge("not blocked", ""),
+      " ",
+      m.list ? ["by blocklist ", h("strong", null, m.list), " (entry ", h("code", null, m.entry), ")"]
+        : m.result !== "none" ? ["by your rule for ", h("code", null, ruleName(m.entry)), m.entry.startsWith("*.") ? " (names below it)" : ""] : null))),
+    blocked && writable ? h("p", null, h("button", { class: "small", onclick: () => addRule(name, "allow", false, []) }, "Allow " + name),
+      h("span", { class: "muted small" }, " adds an allow rule for it and the names below it, which beats every list.")) : null);
+}
+
+const ruleName = (entry) => entry.replace(/^\*\./, "");
+
+async function addRule(name, action, exact, views, comment) {
+  try {
+    const r = await change("POST", "/blocking/rules", { name, action, exact, views, comment: comment || "" });
+    toast(describeResult(r));
+    rerender();
+    return true;
+  } catch (e) { toast(e.message, "err", 8000); return false; }
+}
+
+// viewPicker returns checkboxes for the views (none ticked: automatic).
+function viewPicker(views, autoViews, selected) {
+  if (!views.length) return { el: null, value: () => [] };
+  const boxes = views.map((v) => checkbox(v.name, (selected || []).includes(v.name)));
+  const hint = autoViews && autoViews.length ? "none ticked: " + autoViews.join(", ") + " (every view that offers recursion)" : "none ticked: every view that offers recursion";
+  return {
+    el: h("div", { class: "f" }, h("label", null, "Views"), h("div", { class: "actions" }, boxes, h("span", { class: "muted small" }, hint))),
+    value: () => views.filter((v, i) => boxes[i].input.checked).map((v) => v.name),
+  };
+}
+
+function viewsCell(x) {
+  if (!x.views.length) return h("span", { class: "muted" }, "none");
+  return x.auto_views ? [x.views.join(", "), h("span", { class: "muted small" }, " (auto)")] : x.views.join(", ");
+}
+
+function rulesPanel(rules, views, autoViews, writable) {
+  const name = input({ placeholder: "ads.example.com", class: "mono" });
+  const action = select([["block", "Block"], ["allow", "Allow"]], "block");
+  const exact = checkbox("only this exact name", false);
+  const comment = input({ placeholder: "optional" });
+  const vp = viewPicker(views, autoViews);
+  const err = h("div");
+  const form = h("form", { class: "inline-form", onsubmit: async (ev) => {
+    ev.preventDefault();
+    try {
+      const r = await change("POST", "/blocking/rules", { name: name.value.trim(), action: action.value, exact: exact.input.checked, views: vp.value(), comment: comment.value });
+      toast(describeResult(r));
+      rerender();
+    } catch (e) { mount(err, errorBox(e)); }
+  } },
+    h("div", { class: "f" }, h("label", null, "Action"), action),
+    h("div", { class: "f grow" }, h("label", null, "Domain"), name),
+    h("div", { class: "f" }, h("label", null, " "), exact),
+    h("div", { class: "f" }, h("label", null, "Comment"), comment),
+    vp.el,
+    h("div", { class: "f" }, h("label", null, " "), h("button", { class: "primary", type: "submit" }, "Add rule")));
+  return h("div", { class: "panel" },
+    h("h2", null, "Your rules"),
+    h("p", { class: "muted small" }, "A rule covers the domain and every name below it, unless it's for the exact name only. Rules beat every blocklist, and the most specific rule wins: allowing cdn.example.com overrides blocking example.com for cdn.example.com and the names below it."),
+    writable ? [form, err] : null,
+    table([
+      { label: "Action", render: (r) => [r.action === "block" ? badge("block", "err") : badge("allow", "ok"), r.pending ? [" ", pendingBadge(r.pending)] : null] },
+      { label: "Domain", render: (r) => [h("span", { class: "mono" }, r.name), r.exact ? null : h("span", { class: "muted small" }, " and below")] },
+      views.length ? { label: "Views", render: viewsCell } : null,
+      { label: "Comment", render: (r) => h("span", { class: "muted" }, r.comment || "") },
+      writable ? { label: "", render: (r) => r.pending === "delete" ? "" : h("button", { class: "small danger", onclick: () => del("/blocking/rules/" + r.id, "the rule for " + r.name) }, "Delete"), tdClass: "actions-cell" } : null,
+    ].filter(Boolean), rules, { empty: "No rules yet.", rowClass: (r) => r.pending ? "pending-" + r.pending : "" }));
+}
+
+function listsPanel(lists, views, autoViews, writable) {
+  const name = input({ placeholder: "hagezi-multi", class: "mono", style: { width: "11em" } });
+  const url = input({ placeholder: "https://…", class: "mono" });
+  const vp = viewPicker(views, autoViews);
+  const err = h("div");
+  const form = h("form", { class: "inline-form", onsubmit: async (ev) => {
+    ev.preventDefault();
+    try {
+      const r = await change("POST", "/blocking/lists", { name: name.value.trim(), url: url.value.trim(), views: vp.value() });
+      toast(describeResult(r) + (r.applied ? " — jarld downloads it now" : ""));
+      rerender();
+    } catch (e) { mount(err, errorBox(e)); }
+  } },
+    h("div", { class: "f" }, h("label", null, "Name"), name),
+    h("div", { class: "f grow" }, h("label", null, "URL"), url),
+    vp.el,
+    h("div", { class: "f" }, h("label", null, " "), h("button", { class: "primary", type: "submit" }, "Add list")));
+  const used = new Set(lists.map((l) => l.url));
+  const picks = suggestions.filter(([, u]) => !used.has(u));
+  const totalMB = lists.reduce((a, l) => a + (l.memory_mb || 0), 0);
+  return h("div", { class: "panel" },
+    h("h2", null, "Blocklists"),
+    h("p", { class: "muted small" }, "jarld downloads each list, checks it for changes daily, and keeps using the previous version if an update fails. Hosts, plain domain and Adblock (||domain^) lists work; Adblock entries also cover the names below them."),
+    writable ? [form, err] : null,
+    writable && picks.length ? h("p", { class: "small" }, h("span", { class: "muted" }, "Well-known lists: "),
+      picks.map(([n, u, d], i) => [i ? " · " : "", h("a", { href: "#", title: d + "\n" + u, onclick: (ev) => { ev.preventDefault(); name.value = n; url.value = u; name.focus(); } }, n)])) : null,
+    table([
+      { label: "Name", render: (l) => [h("strong", { class: "mono" }, l.name), l.pending ? [" ", pendingBadge(l.pending)] : null] },
+      { label: "State", render: (l) => { const [t, k] = stateBadge[l.state] || [l.state, ""]; return [badge(t, k), l.status && l.status.error ? h("div", { class: "small err-text" }, l.status.error) : null]; } },
+      { label: "Domains", render: (l) => l.status && l.status.updated_at ? l.status.entries.toLocaleString() : "–", tdClass: "right" },
+      views.length ? { label: "Views", render: viewsCell } : null,
+      { label: "Updated", render: (l) => l.status && l.status.updated_at ? h("span", { title: "checked " + time(l.status.checked_at) }, relTime(l.status.updated_at)) : "–", tdClass: "small" },
+      { label: "Memory", render: (l) => l.memory_mb ? h("span", { title: "named's memory for this list, estimated (all its views)" }, "~" + l.memory_mb + " MB") : "–", tdClass: "small right" },
+      { label: "URL", render: (l) => h("a", { href: l.url, target: "_blank", rel: "noopener", class: "small mono", title: l.url }, shortURL(l.url)) },
+      writable ? { label: "", render: (l) => l.pending === "delete" ? "" : h("span", null,
+        h("button", { class: "small", disabled: l.state === "disabled" || !!l.pending, title: "Download now", onclick: () => refresh(l) }, "Update"), " ",
+        h("button", { class: "small", onclick: () => toggle(l) }, l.disabled ? "Enable" : "Disable"), " ",
+        h("button", { class: "small danger", onclick: () => del("/blocking/lists/" + l.id, "blocklist " + l.name) }, "Delete")), tdClass: "actions-cell" } : null,
+    ].filter(Boolean), lists, { empty: "No blocklists yet.", rowClass: (l) => l.pending ? "pending-" + l.pending : l.disabled ? "disabled" : "" }),
+    totalMB ? h("p", { class: "muted small" }, "named needs about " + totalMB + " MB of memory for these lists (an estimate). A list costs memory in every view it applies to, so pick views if some don't need it.") : null);
+}
+
+function shortURL(u) {
+  const s = u.replace(/^https?:\/\//, "");
+  return s.length > 48 ? s.slice(0, 22) + "…" + s.slice(-24) : s;
+}
+
+async function refresh(l) {
+  toast("Downloading " + l.name + "…", "ok", 2000);
+  try {
+    const r = await api("POST", "/blocking/lists/" + l.id + "/refresh");
+    if (r.status && r.status.error) toast(l.name + ": " + r.status.error, "err", 10000);
+    else toast(l.name + ": " + r.status.entries.toLocaleString() + " domains");
+    rerender();
+  } catch (e) { toast(e.message, "err", 8000); }
+}
+
+async function toggle(l) {
+  try {
+    const r = await change("PATCH", "/blocking/lists/" + l.id, { disabled: !l.disabled });
+    toast(describeResult(r));
+    rerender();
+  } catch (e) { toast(e.message, "err", 8000); }
+}
+
+async function del(path, what) {
+  if (!(await confirmDialog("Delete", "Delete " + what + "?", "Delete", "danger"))) return;
+  try {
+    const r = await change("DELETE", path);
+    toast(describeResult(r));
+    rerender();
+  } catch (e) { toast(e.message, "err", 8000); }
+}
+

internal/web/static/assets/pages/dashboard.js

@@ -40,6 +40,12 @@ export async function render(el) {
       h("strong", null, "The last backup failed "), time(b.last_error_at), ": " + b.last_error,
       h("div", { class: "small" }, "See 'journalctl -u jarl-backup' on the server.")));
   }
+  for (const e of st.blocklist_errors || []) {
+    parts.push(h("div", { class: "msg warn" },
+      h("strong", null, "Blocklist " + e.name + " couldn't be updated: "), e.error + ". ",
+      e.updated_at ? ["named keeps using the version from ", time(e.updated_at), ". "] : "It isn't in use yet. ",
+      "jarld tries again within the hour. ", h("a", { href: "#/blocking" }, "Blocking")));
+  }
   const tf = st.totp;
   if (tf && tf.missing && tf.missing.length) {
     parts.push(h("div", { class: "msg warn" },

internal/web/static/assets/pages/stats.js

@@ -56,7 +56,8 @@ export async function render(el, params, query) {
     tile("Per second", s.avg_qps < 10 ? s.avg_qps.toFixed(2) : Math.round(s.avg_qps).toLocaleString(), "on average"),
     s.peak ? tile("Busiest " + s.peak.per, compact(s.peak.queries), fmtTime(s.peak.time)) : null,
     s.nxdomain_ratio != null ? tile("NXDOMAIN", pct(s.nxdomain_ratio), "of queries named no existing name") : null,
-    s.cache_hit_ratio != null ? tile("Cache hits", pct(s.cache_hit_ratio), "of recursive queries answered from cache") : null));
+    s.cache_hit_ratio != null ? tile("Cache hits", pct(s.cache_hit_ratio), "of recursive queries answered from cache") : null,
+    s.blocked_ratio != null ? tile("Blocked", pct(s.blocked_ratio), compact(s.blocked) + " queries, by your rules and blocklists") : null));
 
   // Volume over time: one series, so no legend box; the title names it.
   const q = timeline.series.find((x) => x.key === "queries") || { values: timeline.times.map(() => 0) };
@@ -84,7 +85,8 @@ export async function render(el, params, query) {
       listCard("Busiest clients", "", barList(t.client)),
       listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + "." })),
       listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + "." })),
-      t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null));
+      t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null,
+      t.blocked.length ? listCard("Most blocked names", "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: (k) => "#/blocking?check=" + encodeURIComponent(k) })) : null));
   } else {
     parts.push(h("div", { class: "msg info" }, "Top names and clients need the response log: set stats.query_log: true in /etc/jarl/jarl.yaml and restart jarl."));
   }
@@ -98,7 +100,8 @@ export async function render(el, params, query) {
     b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel })) : null,
     b.zone.length ? listCard("Zones", "queries per local zone; other = names in no local zone (recursion)", barList(b.zone, { label: (k) => (k === "-" ? "other" : k), href: (k) => (k === "-" ? null : "#/search?q=" + encodeURIComponent(k)) })) : null,
     b.answer.length ? listCard("Answers", "authoritative (your zones) or recursive (from the cache or the internet)", barList(b.answer)) : null,
-    rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null));
+    rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null,
+    b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: () => "#/blocking" })) : null));
   el.classList.remove("refreshing");
   mount(el, parts);
   scheduleRefresh(el, params, query, gen);
@@ -111,10 +114,10 @@ async function loadAll(since, heatSince, queryLog) {
     errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(",") }),
     heat: get("/stats/heatmap", { since: heatSince }),
   };
-  const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt"];
+  const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt", "blocklist"];
   groups.forEach((g) => { p["b_" + g] = get("/stats/breakdown", { group: g, since }); });
   if (queryLog) {
-    for (const k of ["name", "client", "nxdomain", "refused", "servfail"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 });
+    for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 });
   }
   const keys = Object.keys(p);
   const vals = await Promise.all(Object.values(p));