Add blocking: blocklists and block/allow rules with BIND response policy zones

jarld downloads blocklists (Adblock, hosts or plain domains), checks them
daily and reloads named only when a list changed; your own block and
allow rules take precedence, and the most specific rule wins. Lists and
rules apply per view (default: every view that offers recursion). Web UI
Blocking page with name checks, jarl blocklist and jarl block, the
/blocking API, blocked counts in the statistics. Database schema 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adamandClaude Opus 5.5 committed 2026-10-10 09:24:19 +00:00
1 parent 436cd79ec9
commit 9fc23310bd
54 files changed
+4269 -85

No files matched your search

+231
View File
@@ -0,0 +1,231 @@
package engine
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"jarl/internal/blocking"
"jarl/internal/model"
)
func blockRule(name, action string, exact bool, views ...string) Request {
in := BlockRuleInput{Name: &name, Action: &action, Exact: &exact}
if len(views) > 0 {
in.Views = &views
}
return NewRequest(OpBlockRuleCreate, 0, in)
}
func blocklist(name, url string, views ...string) Request {
in := BlocklistInput{Name: &name, URL: &url}
if len(views) > 0 {
in.Views = &views
}
return NewRequest(OpBlocklistCreate, 0, in)
}
// listServer serves a blocklist with an ETag, and counts the downloads
// that sent content.
type listServer struct {
body atomic.Value
full atomic.Int32
calls atomic.Int32
}
func newListServer(t *testing.T, body string) (*listServer, *httptest.Server) {
ls := &listServer{}
ls.body.Store(body)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ls.calls.Add(1)
b := ls.body.Load().(string)
etag := `"` + string(rune('a'+len(b)%26)) + "x" + string(rune('a'+len(b)/26%26)) + `"`
if r.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("ETag", etag)
ls.full.Add(1)
w.Write([]byte(b))
}))
t.Cleanup(srv.Close)
return ls, srv
}
func TestBlockingRender(t *testing.T) {
env := newEnv(t)
setupZone(env)
env.submit(env.admin,
NewRequest(OpViewCreate, 0, ViewInput{Name: model.Ptr("internal"), MatchClients: &[]string{"10.0.0.0/8"}, AdoptZones: true}),
NewRequest(OpViewCreate, 0, ViewInput{Name: model.Ptr("external")}),
)
st := env.state()
ext := st.ViewByName("external")
env.submit(env.admin, NewRequest(OpOptionCreate, 0, OptionInput{Scope: model.Ptr(model.ScopeView), ScopeID: &ext.ID, Name: model.Ptr("recursion"), Value: model.Ptr("no")}))
_, srv := newListServer(t, "||ads.example^\n")
res := env.submit(env.admin,
blocklist("ads", srv.URL+"/list.txt"),
blockRule("tracker.example", "block", false),
blockRule("cdn.tracker.example", "allow", true),
)
if res.Ops[1].Summary != "block tracker.example (and below)" || res.Ops[2].Summary != "allow cdn.tracker.example" {
t.Errorf("summaries: %q, %q", res.Ops[1].Summary, res.Ops[2].Summary)
}
conf, _ := os.ReadFile(filepath.Join(env.dir, "etc/named.conf"))
c := string(conf)
// Automatic views: only internal offers recursion.
internal := c[strings.Index(c, `view "internal"`):strings.Index(c, `view "external"`)]
for _, want := range []string{
"response-policy {\n\t\tzone \"rules.jarl.invalid\" min-update-interval 0 log no;\n\t\tzone \"ads.blocklist.jarl.invalid\" min-update-interval 0 log no;\n\t} qname-wait-recurse no break-dnssec yes;",
`zone "rules.jarl.invalid" {`,
"allow-query { none; };",
} {
if !strings.Contains(internal, want) {
t.Errorf("view internal lacks %q:\n%s", want, internal)
}
}
if strings.Contains(c[strings.Index(c, `view "external"`):], "response-policy") {
t.Error("view external, without recursion, got blocking")
}
rules, err := os.ReadFile(filepath.Join(env.dir, "zones/internal/rules.jarl.invalid.rpz"))
if err != nil || !strings.Contains(string(rules), "cdn.tracker.example\tCNAME\trpz-passthru.\n*.cdn.tracker.example\tCNAME\t.\n") {
t.Errorf("rules zone: %v\n%s", err, rules)
}
// The list's file exists as an empty policy zone until downloaded.
b := env.state().BlocklistByName("ads")
listFile := filepath.Join(env.dir, "cache/blocklists/list-"+strconv.FormatInt(b.ID, 10)+".rpz")
if f, err := os.ReadFile(listFile); err != nil || !strings.Contains(string(f), "not downloaded yet") {
t.Errorf("placeholder: %v %s", err, f)
}
// A view a list or rule names can't be deleted.
res = env.submit(env.admin, NewRequest(OpBlocklistUpdate, b.ID, BlocklistInput{Views: &[]string{"external"}}))
if len(res.Problems) != 1 || res.Problems[0].Code != "blocking_no_recursion" {
t.Errorf("a list in a view without recursion: %+v", res.Problems)
}
if e := env.submitErr(env.admin, NewRequest(OpViewDelete, ext.ID, DeleteInput{Cascade: true})); e.Kind != KindConflict || !strings.Contains(e.Message, "blocklist ads") {
t.Errorf("deleting a used view: %+v", e)
}
st = env.state()
// A second rule for the same name and view is refused.
if e := env.submitErr(env.admin, blockRule("tracker.example", "allow", true)); e.Kind != KindValidation || !strings.Contains(e.Message, "two rules for tracker.example") {
t.Errorf("duplicate rule: %+v", e)
}
// So is a response-policy option where Jarl writes one.
in := st.ViewByName("internal")
if e := env.submitErr(env.admin, NewRequest(OpOptionCreate, 0, OptionInput{Scope: model.Ptr(model.ScopeView), ScopeID: &in.ID, Name: model.Ptr("response-policy"), Value: model.Ptr(`{ zone "x"; }`)})); e.Kind != KindValidation {
t.Errorf("response-policy option: %+v", e)
}
// Bad input.
for _, req := range []Request{blocklist("Bad Name", srv.URL), blocklist("ok", "ftp://x/list"), blockRule("com", "block", false), blockRule("a.example", "deny", false), blockRule("a.example", "block", false, "nope")} {
if e := env.submitErr(env.admin, req); e.Kind != KindBadRequest {
t.Errorf("%s: %+v", req.Data, e)
}
}
}
func TestBlocklistRefresh(t *testing.T) {
env := newEnv(t)
setupZone(env)
ls, srv := newListServer(t, "# test\n||ads.example^\n0.0.0.0 tracker.example\nnot a domain line\n")
env.submit(env.admin, blocklist("ads", srv.URL), blockRule("good.ads.example", "allow", false))
b := env.state().BlocklistByName("ads")
ctx := context.Background()
env.e.RefreshDue(ctx)
s, _ := env.st.BlocklistStatuses(ctx)
st := s[b.ID]
if st == nil || st.Error != "" || st.Entries != 2 || st.Records != 3 || st.Skipped != 1 || st.UpdatedAt == nil {
t.Fatalf("status after the first download: %+v", st)
}
file := env.e.cfg.Paths.BlocklistFile(b.ID)
if f, _ := os.ReadFile(file); !strings.Contains(string(f), "*.ads.example\tCNAME\t.\n") || !strings.Contains(string(f), "tracker.example\tCNAME\t.\n") {
t.Errorf("list zone:\n%s", f)
}
// Not due again: nothing is fetched.
env.e.RefreshDue(ctx)
if ls.calls.Load() != 1 {
t.Errorf("downloads after a second RefreshDue: %d", ls.calls.Load())
}
// A cleared cache leaves an empty placeholder: downloaded again at once.
os.Remove(file)
env.e.RefreshDue(ctx)
if ls.calls.Load() != 2 {
t.Errorf("downloads after the file was lost: %d", ls.calls.Load())
}
if f, _ := os.ReadFile(file); !strings.Contains(string(f), "*.ads.example\tCNAME\t.\n") {
t.Errorf("list zone after the file was lost:\n%s", f)
}
// Unchanged on the server: 304, no rewrite.
firstHash := st.SHA256
fullBefore := ls.full.Load()
env.e.RefreshBlocklist(ctx, b.ID, false)
if ls.full.Load() != fullBefore {
t.Errorf("a conditional download fetched the list again")
}
// Changed: rewritten.
ls.body.Store("||ads.example^\n||more.example^\n")
st, err := env.e.RefreshBlocklist(ctx, b.ID, false)
if err != nil || st.Entries != 2 || st.Records != 4 || st.SHA256 == firstHash {
t.Errorf("after a change: %+v, %v", st, err)
}
// A broken download keeps the previous version and reports why.
before, _ := os.ReadFile(file)
ls.body.Store("<html><body>Not found</body></html>\n")
st, err = env.e.RefreshBlocklist(ctx, b.ID, true)
if err == nil || !strings.Contains(st.Error, "no domains found") {
t.Errorf("garbage: %+v, %v", st, err)
}
if after, _ := os.ReadFile(file); string(after) != string(before) {
t.Error("a failed update changed the list's file")
}
// A failed attempt is retried after RetryAfter, not at every check.
calls := ls.calls.Load()
env.e.RefreshDue(ctx)
if ls.calls.Load() != calls {
t.Error("a failed list was tried again before RetryAfter")
}
// Too big.
env.e.Limits.MaxBytes = 10
ls.body.Store("||ads.example^\n||more.example^\n||third.example^\n")
if st, _ = env.e.RefreshBlocklist(ctx, b.ID, true); !strings.Contains(st.Error, "larger than") {
t.Errorf("size limit: %+v", st)
}
env.e.Limits = DefaultBlocklistLimits()
env.e.RefreshBlocklist(ctx, b.ID, true)
// What blocks what.
check := func(name string) blocking.Match {
m, err := env.e.CheckBlocking(env.state(), name)
if err != nil || len(m) != 1 {
t.Fatalf("check %s: %+v %v", name, m, err)
}
return m[0]
}
if m := check("x.ads.example"); m.Result != "blocked" || m.List != "ads" || m.Entry != "*.ads.example" {
t.Errorf("x.ads.example: %+v", m)
}
if m := check("a.good.ads.example"); m.Result != "allowed" || m.RuleID == 0 || m.Entry != "*.good.ads.example" {
t.Errorf("a.good.ads.example: %+v", m)
}
if m := check("fine.example"); m.Result != "none" {
t.Errorf("fine.example: %+v", m)
}
// Deleting the list removes its file and status.
env.submit(env.admin, NewRequest(OpBlocklistDelete, b.ID, nil))
env.e.RefreshDue(ctx)
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Errorf("the deleted list's file is still there: %v", err)
}
if s, _ := env.st.BlocklistStatuses(ctx); s[b.ID] != nil {
t.Error("the deleted list's status is still there")
}
}
+403
View File
@@ -0,0 +1,403 @@
package engine
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"jarl/internal/blocking"
"jarl/internal/model"
"jarl/internal/render"
"jarl/internal/version"
)
// BlocklistLimits bound blocklist downloads (blocking in jarl.yaml).
type BlocklistLimits struct {
Refresh time.Duration // how often a list is checked for changes
RetryAfter time.Duration // after a failed attempt
MaxBytes int64 // per download
MaxEntries int // domains per list
Timeout time.Duration // per download
}
// DefaultBlocklistLimits are the limits without configuration.
func DefaultBlocklistLimits() BlocklistLimits {
return BlocklistLimits{Refresh: 24 * time.Hour, RetryAfter: time.Hour, MaxBytes: 64 << 20, MaxEntries: 1_000_000, Timeout: 2 * time.Minute}
}
// KickBlocklists asks the download loop to look for work now (a list was
// added or changed).
func (e *Engine) KickBlocklists() {
select {
case e.blKick <- struct{}{}:
default:
}
}
// BlocklistKicks delivers KickBlocklists' signals to the download loop.
func (e *Engine) BlocklistKicks() <-chan struct{} { return e.blKick }
// ensureBlocklistFiles creates missing blocklist zone files as empty
// policy zones, so named can load its configuration before a list has
// been downloaded. Existing files are never touched.
func (e *Engine) ensureBlocklistFiles(paths []string) error {
var errs []error
for _, path := range paths {
if _, err := os.Stat(path); err == nil {
continue
}
if err := e.deploy.writeNew(path, []byte(blocking.Placeholder()), 0o640); err != nil && !errors.Is(err, fs.ErrExist) {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
// RefreshDue downloads the blocklists that are due: never tried, URL
// changed, file missing or empty (the cache was cleared), last check older
// than Refresh, or a failed attempt older than RetryAfter. Lists that are
// disabled or apply to no view are skipped. It also removes the files and
// status of deleted lists.
func (e *Engine) RefreshDue(ctx context.Context) {
st, err := e.store.LoadState(ctx)
if err != nil {
e.log.Warn("blocklists: loading the configuration failed", "err", err)
return
}
statuses, err := e.store.BlocklistStatuses(ctx)
if err != nil {
e.log.Warn("blocklists: loading their status failed", "err", err)
return
}
out := render.Render(st, e.cfg.Paths, e.cfg.Extras)
if err := e.ensureBlocklistFiles(out.Blocklists); err != nil {
e.log.Warn("blocklists: creating empty zone files failed", "err", err)
}
e.cleanBlocklists(ctx, st, statuses)
now := e.Now()
for _, b := range st.SortedBlocklists() {
if b.Disabled || len(st.BlockingViews(b.ViewIDs)) == 0 {
continue
}
s := statuses[b.ID]
due := s == nil || s.URL != b.URL || s.CheckedAt == nil
if !due {
wait := e.Limits.Refresh
if s.Error != "" {
wait = e.Limits.RetryAfter
}
due = now.Sub(*s.CheckedAt) >= wait || s.Error == "" && isPlaceholder(e.cfg.Paths.BlocklistFile(b.ID))
}
if !due {
continue
}
if ctx.Err() != nil {
return
}
if _, err := e.RefreshBlocklist(ctx, b.ID, false); err != nil && ctx.Err() == nil {
e.log.Warn("blocklist update failed; named keeps using the previous version", "list", b.Name, "err", err)
}
}
}
// isPlaceholder reports whether a list's zone file is missing or still the
// empty policy zone written before its download.
func isPlaceholder(path string) bool {
f, err := os.Open(path)
if err != nil {
return true
}
defer f.Close()
head := make([]byte, len(blocking.Placeholder()))
n, _ := io.ReadFull(f, head)
return string(head[:n]) == blocking.Placeholder()
}
// cleanBlocklists removes the zone files and status of deleted lists.
func (e *Engine) cleanBlocklists(ctx context.Context, st *model.State, statuses map[int64]*model.BlocklistStatus) {
for id := range statuses {
if st.Blocklists[id] == nil {
e.store.DeleteBlocklistStatus(ctx, id)
}
}
dir := e.cfg.Paths.BlocklistsDir()
entries, err := os.ReadDir(dir)
if err != nil {
return
}
for _, ent := range entries {
n := ent.Name()
idText, ok := strings.CutPrefix(strings.TrimSuffix(n, ".rpz"), "list-")
id, err := strconv.ParseInt(idText, 10, 64)
if !ok || !strings.HasSuffix(n, ".rpz") || err != nil || st.Blocklists[id] != nil {
continue
}
if err := os.Remove(filepath.Join(dir, n)); err == nil {
e.log.Info("removed the zone file of a deleted blocklist", "file", n)
}
}
}
// errNotModified means the server said the list hasn't changed.
var errNotModified = errors.New("not modified")
// RefreshBlocklist downloads a list now and, when its content changed,
// replaces its zone file and makes named reload it. force downloads even
// when the server says nothing changed. A failure keeps the previous
// version. It returns the new status.
func (e *Engine) RefreshBlocklist(ctx context.Context, id int64, force bool) (*model.BlocklistStatus, error) {
e.blMu.Lock()
defer e.blMu.Unlock()
st, err := e.store.LoadState(ctx)
if err != nil {
return nil, err
}
b := st.Blocklists[id]
if b == nil {
return nil, errf(KindNotFound, "blocklist %d does not exist", id)
}
statuses, err := e.store.BlocklistStatuses(ctx)
if err != nil {
return nil, err
}
prev := statuses[id]
if prev == nil {
prev = &model.BlocklistStatus{ID: id}
}
file := e.cfg.Paths.BlocklistFile(id)
now := e.Now()
next := *prev
next.CheckedAt = &now
conditional := !force && prev.URL == b.URL && prev.SHA256 != "" && !isPlaceholder(file)
err = e.downloadBlocklist(ctx, st, b, &next, conditional, file)
next.URL, next.Error = b.URL, ""
if err != nil && !errors.Is(err, errNotModified) {
next.Error = err.Error()
}
if serr := e.store.SetBlocklistStatus(ctx, &next); serr != nil && err == nil {
err = serr
}
if errors.Is(err, errNotModified) {
err = nil
}
return &next, err
}
// downloadBlocklist fetches, parses, checks and installs a list, filling
// in s. It returns errNotModified when there is nothing to do.
func (e *Engine) downloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist, s *model.BlocklistStatus, conditional bool, file string) error {
ctx, cancel := context.WithTimeout(ctx, e.Limits.Timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.URL, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Jarl/"+version.Short()+" (blocklist download)")
if conditional {
if s.ETag != "" {
req.Header.Set("If-None-Match", s.ETag)
}
if s.LastModified != "" {
req.Header.Set("If-Modified-Since", s.LastModified)
}
}
client := e.HTTP
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("download failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified && conditional {
return errNotModified
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download failed: the server answered %s", resp.Status)
}
zone := blocking.ListZone(b.Name)
body := &limitedReader{r: resp.Body, left: e.Limits.MaxBytes}
l, err := blocking.Parse(body, blocking.Limits{MaxEntries: e.Limits.MaxEntries, MaxNameLen: 253 - len(zone) - 3})
if body.over {
return fmt.Errorf("the list is larger than %d MB (blocking.max_download_mb in jarl.yaml)", e.Limits.MaxBytes>>20)
}
if errors.Is(err, blocking.ErrTooManyEntries) {
return fmt.Errorf("%v (blocking.max_entries in jarl.yaml)", err)
}
if err != nil {
return fmt.Errorf("download failed: %w", err)
}
if len(l.Domains) == 0 {
return fmt.Errorf("no domains found in %d lines: is this a blocklist in hosts, domains or Adblock format?", l.Lines)
}
entries, skipped := len(l.Domains), l.Skipped
blocking.Compact(l)
dir := filepath.Dir(file)
if err := e.deploy.EnsureDir(dir, e.deploy.dirMode(dir)); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, ".list-*.rpz")
if err != nil {
return err
}
tmpName := tmp.Name()
defer os.Remove(tmpName) // after a successful rename, there's nothing to remove
records, hash, err := blocking.FeedZone(tmp, fmt.Sprintf("Blocklist %s from %s, %s", b.Name, b.URL, e.Now().UTC().Format(time.RFC3339)), uint32(e.Now().Unix()), l)
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err != nil {
return err
}
s.ETag, s.LastModified = resp.Header.Get("ETag"), resp.Header.Get("Last-Modified")
s.Entries, s.Skipped = entries, skipped
if hash == s.SHA256 && s.URL == b.URL && !isPlaceholder(file) {
return errNotModified // same content: named keeps what it has
}
if e.tools != nil && !e.cfg.SkipCheck {
res, err := e.tools.CheckZone(ctx, zone, tmpName)
if err != nil {
return fmt.Errorf("running named-checkzone failed: %w", err)
}
if !res.OK {
msg := "named-checkzone rejected the list's zone"
for _, m := range res.Messages {
if m.Error {
msg += ": " + m.Text
break
}
}
return errors.New(msg)
}
}
if err := e.deploy.finish(tmpName, file, 0o640); err != nil {
return err
}
now := e.Now()
s.UpdatedAt, s.Records, s.SHA256 = &now, records, hash
e.log.Info("blocklist updated", "list", b.Name, "domains", entries, "records", records, "skipped_lines", skipped)
return e.reloadBlocklist(ctx, st, b)
}
// reloadBlocklist makes named load a list's new file in every view that
// uses it.
func (e *Engine) reloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist) error {
if e.cfg.NoReload || e.tools == nil {
return nil
}
if adopted, err := e.Adopted(); err != nil || !adopted {
return nil // the file is ready for when named loads Jarl's configuration
}
for _, z := range render.PolicyZones(st, e.cfg.Paths) {
if z.ListID != b.ID {
continue
}
if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil {
return fmt.Errorf("the list was downloaded, but named didn't reload it: %w", err)
}
}
return nil
}
// limitedReader reads at most left bytes and notes when there was more.
type limitedReader struct {
r io.Reader
left int64
over bool
}
func (l *limitedReader) Read(p []byte) (int, error) {
if l.left <= 0 {
var one [1]byte
if n, _ := l.r.Read(one[:]); n > 0 {
l.over = true
}
return 0, io.EOF
}
if int64(len(p)) > l.left {
p = p[:l.left]
}
n, err := l.r.Read(p)
l.left -= int64(n)
return n, err
}
// CheckBlocking reports, for every view with blocking, whether name is
// blocked or allowed there, and by which rule or list. It looks names up
// the way named does: rules first, then the lists in order.
func (e *Engine) CheckBlocking(st *model.State, name string) ([]blocking.Match, error) {
n, err := blocking.NormalizeName(name)
if err != nil {
return nil, errf(KindBadRequest, "%v", err)
}
var out []blocking.Match
feeds := map[string]string{} // file -> matching entry ("" none)
var cur *blocking.Match
for _, z := range render.PolicyZones(st, e.cfg.Paths) {
if cur == nil || cur.View != z.View {
out = append(out, blocking.Match{View: z.View, Result: "none"})
cur = &out[len(out)-1]
}
if cur.Result != "none" {
continue // an earlier zone decided
}
if z.ListID == 0 {
owner, act, ok := blocking.MatchRules(blocking.RuleEntries(render.ViewRules(st, z.ViewID)), n)
if ok {
cur.Entry, cur.RuleID = owner, ruleFor(st, z.ViewID, owner)
cur.Result = map[string]string{blocking.ActionBlock: "blocked", blocking.ActionAllow: "allowed"}[act]
}
continue
}
entry, seen := feeds[z.File]
if !seen {
owner, ok, err := blocking.MatchFeed(z.File, n)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
if ok {
entry = owner
}
feeds[z.File] = entry
}
if entry != "" {
cur.Result, cur.Entry, cur.List = "blocked", entry, st.Blocklists[z.ListID].Name
}
}
return out, nil
}
// ruleFor finds the rule behind a rules-zone entry: the rule for the
// entry's name, or (for entries added so the most specific rule wins) the
// closest rule above it that covers the names below it.
func ruleFor(st *model.State, viewID int64, owner string) int64 {
name := strings.TrimPrefix(owner, "*.")
wild := name != owner
var best *model.BlockRule
for _, r := range st.SortedBlockRules() {
if !slices.Contains(st.BlockingViews(r.ViewIDs), viewID) {
continue
}
if r.Name == name && (!wild || !r.Exact) {
return r.ID
}
if !r.Exact && strings.HasSuffix(name, "."+r.Name) && (best == nil || len(r.Name) > len(best.Name)) {
best = r
}
}
if best != nil {
return best.ID
}
return 0
}
+272
View File
@@ -0,0 +1,272 @@
package engine
import (
"encoding/json"
"fmt"
"net/url"
"regexp"
"slices"
"strings"
"jarl/internal/auth"
"jarl/internal/blocking"
"jarl/internal/model"
)
// ---- blocking: blocklists and rules (docs/blocking.md)
var blocklistNameRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,38}[a-z0-9])?$`)
func blockingAction() auth.Action { return auth.Action{Resource: auth.ResBlocking, Write: true} }
// viewIDs resolves the views of a blocklist or rule input: names or IDs,
// in view order, without duplicates. nil, nil means "not given".
func viewIDs(st *model.State, names *[]string, ids *[]int64) ([]int64, error) {
if names == nil && ids == nil {
return nil, nil
}
want := map[int64]bool{}
if names != nil {
for _, n := range *names {
v := st.ViewByName(strings.TrimSpace(n))
if v == nil {
return nil, errf(KindBadRequest, "view %q does not exist", n)
}
want[v.ID] = true
}
}
if ids != nil {
for _, id := range *ids {
if st.Views[id] == nil {
return nil, errf(KindBadRequest, "view %d does not exist", id)
}
want[id] = true
}
}
out := []int64{}
for _, v := range st.OrderedViews() {
if want[v.ID] {
out = append(out, v.ID)
}
}
return out, nil
}
// CheckBlocklistURL checks a blocklist URL: http or https, with a host.
func CheckBlocklistURL(raw string) (string, error) {
raw = strings.TrimSpace(raw)
u, err := url.Parse(raw)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return "", fmt.Errorf("%q is not an http or https URL", raw)
}
return raw, nil
}
func applyBlocklistPut(st *model.State, op model.Op, c *applyCtx) error {
if err := c.require(blockingAction()); err != nil {
return err
}
var in BlocklistInput
if err := decode(op.Data, &in); err != nil {
return err
}
var b *model.Blocklist
if op.Kind == OpBlocklistCreate {
if op.ID == 0 || st.Blocklists[op.ID] != nil {
return errf(KindConflict, "blocklist ID %d is already in use", op.ID)
}
if in.Name == nil || in.URL == nil {
return errf(KindBadRequest, "a blocklist needs a name and a URL")
}
b = &model.Blocklist{ID: op.ID, ViewIDs: []int64{}}
} else {
b = st.Blocklists[op.ID]
if b == nil {
return errf(KindNotFound, "blocklist %d does not exist", op.ID)
}
if err := checkVersion(op, b.Version, "blocklist "+b.Name); err != nil {
return err
}
}
if in.Name != nil {
name := strings.ToLower(strings.TrimSpace(*in.Name))
if !blocklistNameRe.MatchString(name) {
return errf(KindBadRequest, "blocklist name %q: use 1 to 40 lowercase letters, digits and hyphens, like hagezi-pro", *in.Name)
}
if other := st.BlocklistByName(name); other != nil && other.ID != b.ID {
return errf(KindConflict, "blocklist %s already exists", name)
}
b.Name = name
}
if in.URL != nil {
u, err := CheckBlocklistURL(*in.URL)
if err != nil {
return errf(KindBadRequest, "%v", err)
}
b.URL = u
}
ids, err := viewIDs(st, in.Views, in.ViewIDs)
if err != nil {
return err
}
if ids != nil {
b.ViewIDs = ids
}
if in.Disabled != nil {
b.Disabled = *in.Disabled
}
if in.Comment != nil {
b.Comment = *in.Comment
}
b.Touch(c.user(), c.now)
st.Blocklists[b.ID] = b
return nil
}
func applyBlocklistDelete(st *model.State, op model.Op, c *applyCtx) error {
if err := c.require(blockingAction()); err != nil {
return err
}
b := st.Blocklists[op.ID]
if b == nil {
return errf(KindNotFound, "blocklist %d does not exist", op.ID)
}
if err := checkVersion(op, b.Version, "blocklist "+b.Name); err != nil {
return err
}
delete(st.Blocklists, b.ID)
return nil
}
func applyBlockRulePut(st *model.State, op model.Op, c *applyCtx) error {
if err := c.require(blockingAction()); err != nil {
return err
}
var in BlockRuleInput
if err := decode(op.Data, &in); err != nil {
return err
}
var r *model.BlockRule
if op.Kind == OpBlockRuleCreate {
if op.ID == 0 || st.BlockRules[op.ID] != nil {
return errf(KindConflict, "rule ID %d is already in use", op.ID)
}
if in.Name == nil {
return errf(KindBadRequest, "a rule needs a domain name")
}
r = &model.BlockRule{ID: op.ID, Action: model.BlockActionBlock, ViewIDs: []int64{}}
} else {
r = st.BlockRules[op.ID]
if r == nil {
return errf(KindNotFound, "rule %d does not exist", op.ID)
}
if err := checkVersion(op, r.Version, "rule "+r.Name); err != nil {
return err
}
}
if in.Name != nil {
n, err := blocking.NormalizeName(*in.Name)
if err != nil {
return errf(KindBadRequest, "%v", err)
}
r.Name = n
}
if in.Action != nil {
a := strings.ToLower(strings.TrimSpace(*in.Action))
if a != model.BlockActionBlock && a != model.BlockActionAllow {
return errf(KindBadRequest, "a rule's action is block or allow, not %q", *in.Action)
}
r.Action = a
}
if in.Exact != nil {
r.Exact = *in.Exact
}
ids, err := viewIDs(st, in.Views, in.ViewIDs)
if err != nil {
return err
}
if ids != nil {
r.ViewIDs = ids
}
if in.Comment != nil {
r.Comment = *in.Comment
}
r.Touch(c.user(), c.now)
st.BlockRules[r.ID] = r
return nil
}
func applyBlockRuleDelete(st *model.State, op model.Op, c *applyCtx) error {
if err := c.require(blockingAction()); err != nil {
return err
}
r := st.BlockRules[op.ID]
if r == nil {
return errf(KindNotFound, "rule %d does not exist", op.ID)
}
if err := checkVersion(op, r.Version, "rule "+r.Name); err != nil {
return err
}
delete(st.BlockRules, r.ID)
return nil
}
// blockingUsers lists the blocklists and rules that name a view.
func blockingUsers(st *model.State, viewID int64) []string {
var out []string
for _, b := range st.SortedBlocklists() {
if slices.Contains(b.ViewIDs, viewID) {
out = append(out, "blocklist "+b.Name)
}
}
for _, r := range st.SortedBlockRules() {
if slices.Contains(r.ViewIDs, viewID) {
out = append(out, ruleLabel(r))
}
}
return out
}
// ruleLabel describes a rule: "block ads.example (and below)".
func ruleLabel(r *model.BlockRule) string {
s := r.Action + " " + r.Name
if !r.Exact {
s += " (and below)"
}
return s
}
// ---- expansion
func (x *expander) blocklistCreate(req Request) ([]model.Op, error) {
return x.simpleCreate(req, OpBlocklistCreate, func(data json.RawMessage) (string, error) {
var in BlocklistInput
if err := decode(data, &in); err != nil {
return "", err
}
if in.Name == nil || in.URL == nil {
return "", errf(KindBadRequest, "a blocklist needs a name and a URL")
}
return "add blocklist " + *in.Name + " (" + *in.URL + ")", nil
})
}
func (x *expander) blockRuleCreate(req Request) ([]model.Op, error) {
return x.simpleCreate(req, OpBlockRuleCreate, func(data json.RawMessage) (string, error) {
var in BlockRuleInput
if err := decode(data, &in); err != nil {
return "", err
}
if in.Name == nil {
return "", errf(KindBadRequest, "a rule needs a domain name")
}
r := &model.BlockRule{Name: strings.ToLower(strings.TrimSuffix(strings.TrimSpace(*in.Name), ".")), Action: model.BlockActionBlock}
if in.Action != nil {
r.Action = strings.ToLower(*in.Action)
}
if in.Exact != nil {
r.Exact = *in.Exact
}
return ruleLabel(r), nil
})
}
+42
View File
@@ -183,6 +183,48 @@ func (d *Deployer) writeAtomic(path string, content []byte, mode fs.FileMode) er
return nil
}
// writeNew creates a file that doesn't exist yet (fs.ErrExist if it
// does), with the deploy group.
func (d *Deployer) writeNew(path string, content []byte, mode fs.FileMode) error {
dir := filepath.Dir(path)
if err := d.EnsureDir(dir, d.dirMode(dir)); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
if err != nil {
return err
}
_, err = f.Write(content)
if gid, gerr := d.groupID(); err == nil && gerr == nil && gid >= 0 {
err = f.Chown(-1, gid)
}
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
os.Remove(path)
}
return err
}
// finish gives a temporary file written next to path its mode and the
// deploy group, and moves it into place.
func (d *Deployer) finish(tmp, path string, mode fs.FileMode) error {
if err := os.Chmod(tmp, mode); err != nil {
return err
}
gid, err := d.groupID()
if err != nil {
return err
}
if gid >= 0 {
if err := os.Chown(tmp, -1, gid); err != nil {
return fmt.Errorf("chgrp %s: %w", path, err)
}
}
return os.Rename(tmp, path)
}
// Install writes files and removes paths, returning a backup that undoes
// the changes. On error the files already written are restored.
func (d *Deployer) Install(files []*render.File, remove []string, dirs []string) (*Backup, error) {
+28
View File
@@ -101,7 +101,35 @@ func DiffOps(from, to *model.State) []model.Op {
add(OpRecordUpdate, k, o.Version, in, fmt.Sprintf("restore %s %s %s", n.Name, n.Type, n.Data))
}
}
for _, k := range sortedKeys(to.Blocklists) {
n := to.Blocklists[k]
in := BlocklistInput{Name: &n.Name, URL: &n.URL, ViewIDs: model.Ptr(append([]int64{}, n.ViewIDs...)), Disabled: &n.Disabled, Comment: &n.Comment}
if o, ok := from.Blocklists[k]; !ok {
add(OpBlocklistCreate, k, 0, in, "add blocklist "+n.Name)
} else if !sameObject(o, n) {
add(OpBlocklistUpdate, k, o.Version, in, "restore blocklist "+n.Name)
}
}
for _, k := range sortedKeys(to.BlockRules) {
n := to.BlockRules[k]
in := BlockRuleInput{Name: &n.Name, Action: &n.Action, Exact: &n.Exact, ViewIDs: model.Ptr(append([]int64{}, n.ViewIDs...)), Comment: &n.Comment}
if o, ok := from.BlockRules[k]; !ok {
add(OpBlockRuleCreate, k, 0, in, "add rule: "+ruleLabel(n))
} else if !sameObject(o, n) {
add(OpBlockRuleUpdate, k, o.Version, in, "restore rule: "+ruleLabel(n))
}
}
// Deletions, most dependent objects first.
for _, k := range sortedKeys(from.BlockRules) {
if _, ok := to.BlockRules[k]; !ok {
add(OpBlockRuleDelete, k, from.BlockRules[k].Version, nil, "delete rule: "+ruleLabel(from.BlockRules[k]))
}
}
for _, k := range sortedKeys(from.Blocklists) {
if _, ok := to.Blocklists[k]; !ok {
add(OpBlocklistDelete, k, from.Blocklists[k].Version, nil, "delete blocklist "+from.Blocklists[k].Name)
}
}
for _, k := range sortedKeys(from.Records) {
if _, ok := to.Records[k]; !ok {
o := from.Records[k]
+63 -1
View File
@@ -14,8 +14,10 @@ import (
"fmt"
"io/fs"
"log/slog"
"net/http"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
@@ -23,6 +25,7 @@ import (
"jarl/internal/auth"
"jarl/internal/bind"
"jarl/internal/blocking"
"jarl/internal/dnsname"
"jarl/internal/model"
"jarl/internal/namedconf"
@@ -64,6 +67,12 @@ type Engine struct {
deploy *Deployer
mu sync.Mutex // serializes applies
Now func() time.Time
// Blocklist downloads (blocklists.go).
blMu sync.Mutex // serializes downloads
blKick chan struct{} // a list was added or changed
HTTP *http.Client // for downloads; nil: a default client
Limits BlocklistLimits
}
// New creates an engine.
@@ -78,6 +87,8 @@ func New(st *store.Store, tools *bind.Tools, cfg Config, log *slog.Logger) *Engi
store: st, tools: tools, cfg: cfg, log: log,
deploy: &Deployer{Group: cfg.FileGroup, NamedWritable: []string{cfg.Paths.ZoneDir, cfg.Paths.CacheDir}},
Now: time.Now,
blKick: make(chan struct{}, 1),
Limits: DefaultBlocklistLimits(),
}
}
@@ -383,6 +394,7 @@ type plan struct {
drift []DriftEntry
configChanged bool
reloadZones []*model.Zone
reloadPolicy []render.PolicyZone // rules zones whose file changed
verify map[int64]uint32
newZones bool
backup *Backup
@@ -446,7 +458,7 @@ func (e *Engine) makePlan(ctx context.Context, base, work *model.State, user str
}
}
for path, df := range deployed {
if pl.out.File(path) != nil || df.Kind != render.KindZone {
if pl.out.File(path) != nil || df.Kind != render.KindZone && df.Kind != render.KindPolicy {
continue
}
old, existed, _ := readFile(path)
@@ -484,6 +496,16 @@ func (e *Engine) makePlan(ctx context.Context, base, work *model.State, user str
if len(work.Zones) != len(base.Zones) {
pl.configChanged = true
}
// Rules zones whose content changed. New ones come with a change to
// named.conf, and reconfig loads them.
for _, z := range pl.out.Policy {
if z.ListID != 0 || !changedPaths[z.File] {
continue
}
if slices.Contains(baseOut.Policy, z) {
pl.reloadPolicy = append(pl.reloadPolicy, z)
}
}
if !e.cfg.SkipCheck {
if err := e.stagingCheck(ctx, pl); err != nil {
@@ -519,6 +541,16 @@ func (e *Engine) stagingCheck(ctx context.Context, pl *plan) error {
return err
}
}
// Blocklists as empty policy zones: their content is checked when
// they are downloaded.
for _, path := range sout.Blocklists {
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return err
}
if err := os.WriteFile(path, []byte(blocking.Placeholder()), 0o640); err != nil {
return err
}
}
main, err := e.stagingMain(dir, sp.MainConfig())
if err != nil {
return err
@@ -796,6 +828,12 @@ func (e *Engine) applyOps(ctx context.Context, cs *model.Changeset, reqs []Reque
res.ChangesetID = cs.ID
res.Applied = true
e.removeDerived(pl)
for _, op := range cs.Ops {
if strings.HasPrefix(op.Kind, "blocklist.") {
e.KickBlocklists()
break
}
}
e.log.Info("changeset applied", "changeset", cs.ID, "user", p.Name(), "ops", len(cs.Ops), "files", len(res.Files), "description", cs.Description)
return res, nil
}
@@ -829,6 +867,9 @@ func (e *Engine) deployAndReload(ctx context.Context, pl *plan, res *Result, rel
return &Error{Kind: KindBIND, Message: "writing configuration files failed (nothing was changed): " + err.Error()}
}
pl.backup = backup
if err := e.ensureBlocklistFiles(pl.out.Blocklists); err != nil {
e.log.Warn("creating a blocklist's empty zone file failed", "err", err)
}
pl.reloaded = reload
if !reload {
return nil
@@ -860,6 +901,11 @@ func (e *Engine) rollback(ctx context.Context, pl *plan) {
e.log.Error("rndc reload after restore failed", "zone", z.Name, "err", err)
}
}
for _, z := range pl.reloadPolicy {
if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil {
e.log.Error("rndc reload after restore failed", "zone", z.Name, "err", err)
}
}
}
// namedView returns the view name to pass to rndc for a zone ("" without
@@ -885,6 +931,16 @@ func (e *Engine) reload(ctx context.Context, pl *plan) ([]string, error) {
}
done = append(done, "reload "+zoneDesc(pl.work, z))
}
for _, z := range pl.reloadPolicy {
label := "blocking rules"
if z.View != "" {
label += " [view " + z.View + "]"
}
if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil {
return done, fmt.Errorf("reloading %s failed: %v", label, err)
}
done = append(done, "reload "+label)
}
// named reloads zones asynchronously and reports load errors only in
// its log, so wait until it serves the new serials.
deadline := time.Now().Add(e.cfg.ReloadTimeout)
@@ -1003,6 +1059,12 @@ func (e *Engine) Resync(ctx context.Context, p *auth.Principal, opts ApplyOption
pl.verify[z.ID] = z.SOA.Serial
}
}
pl.reloadPolicy = nil
for _, z := range pl.out.Policy {
if z.ListID == 0 {
pl.reloadPolicy = append(pl.reloadPolicy, z)
}
}
reload := !e.cfg.NoReload && !opts.NoReload
if adopted, aerr := e.Adopted(); reload && (aerr != nil || !adopted) {
reload = false
+24
View File
@@ -222,6 +222,30 @@ func (x *expander) expand(req Request) ([]model.Op, error) {
return []model.Op{x.versioned(req, o.Version, summary)}, nil
case OpSettingsUpdate:
return []model.Op{newOpRaw(OpSettingsUpdate, 0, 0, req.Data, "update settings")}, nil
case OpBlocklistCreate:
return x.blocklistCreate(req)
case OpBlocklistUpdate, OpBlocklistDelete:
b := x.st.Blocklists[req.ID]
if b == nil {
return nil, errf(KindNotFound, "blocklist %d does not exist", req.ID)
}
verb := "change"
if req.Op == OpBlocklistDelete {
verb = "delete"
}
return []model.Op{x.versioned(req, b.Version, verb+" blocklist "+b.Name)}, nil
case OpBlockRuleCreate:
return x.blockRuleCreate(req)
case OpBlockRuleUpdate, OpBlockRuleDelete:
r := x.st.BlockRules[req.ID]
if r == nil {
return nil, errf(KindNotFound, "rule %d does not exist", req.ID)
}
verb := "change rule: "
if req.Op == OpBlockRuleDelete {
verb = "delete rule: "
}
return []model.Op{x.versioned(req, r.Version, verb+ruleLabel(r))}, nil
}
return nil, errf(KindBadRequest, "unknown operation %q", req.Op)
}
+32 -1
View File
@@ -30,6 +30,13 @@ const (
OpOptionDelete = "option.delete"
OpSettingsUpdate = "settings.update"
OpBlocklistCreate = "blocklist.create"
OpBlocklistUpdate = "blocklist.update"
OpBlocklistDelete = "blocklist.delete"
OpBlockRuleCreate = "blockrule.create"
OpBlockRuleUpdate = "blockrule.update"
OpBlockRuleDelete = "blockrule.delete"
// Requests only: disabling and enabling records, by ID or by match
// (zone, name, type and optionally data), become record.update
// operations.
@@ -40,7 +47,8 @@ const (
// OpKinds lists all operation kinds.
var OpKinds = []string{OpZoneCreate, OpZoneUpdate, OpZoneDelete, OpRecordCreate, OpRecordUpdate, OpRecordDelete, OpRRSetReplace,
OpViewCreate, OpViewUpdate, OpViewDelete, OpViewReorder, OpACLCreate, OpACLUpdate, OpACLDelete,
OpTSIGCreate, OpTSIGUpdate, OpTSIGDelete, OpOptionCreate, OpOptionUpdate, OpOptionDelete, OpSettingsUpdate}
OpTSIGCreate, OpTSIGUpdate, OpTSIGDelete, OpOptionCreate, OpOptionUpdate, OpOptionDelete, OpSettingsUpdate,
OpBlocklistCreate, OpBlocklistUpdate, OpBlocklistDelete, OpBlockRuleCreate, OpBlockRuleUpdate, OpBlockRuleDelete}
// Request is an operation as submitted by a client. Requests are expanded
// into one or more stored operations (for example an A record with "ptr"
@@ -213,3 +221,26 @@ type SettingsInput struct {
type DeleteInput struct {
Cascade bool `json:"cascade,omitempty"`
}
// BlocklistInput creates or updates a blocklist. Views are given by name
// (Views) or ID (ViewIDs); an empty list means every view that offers
// recursion.
type BlocklistInput struct {
Name *string `json:"name,omitempty"`
URL *string `json:"url,omitempty"`
Views *[]string `json:"views,omitempty"`
ViewIDs *[]int64 `json:"view_ids,omitempty"`
Disabled *bool `json:"disabled,omitempty"`
Comment *string `json:"comment,omitempty"`
}
// BlockRuleInput creates or updates a block or allow rule. Views as for
// BlocklistInput.
type BlockRuleInput struct {
Name *string `json:"name,omitempty"`
Action *string `json:"action,omitempty"` // block (default) or allow
Exact *bool `json:"exact,omitempty"` // only the name, not the names below it
Views *[]string `json:"views,omitempty"`
ViewIDs *[]int64 `json:"view_ids,omitempty"`
Comment *string `json:"comment,omitempty"`
}
+11
View File
@@ -98,6 +98,14 @@ func applyOp(st *model.State, op model.Op, c *applyCtx) error {
err = applyOptionDelete(st, op, c)
case OpSettingsUpdate:
err = applySettings(st, op, c)
case OpBlocklistCreate, OpBlocklistUpdate:
err = applyBlocklistPut(st, op, c)
case OpBlocklistDelete:
err = applyBlocklistDelete(st, op, c)
case OpBlockRuleCreate, OpBlockRuleUpdate:
err = applyBlockRulePut(st, op, c)
case OpBlockRuleDelete:
err = applyBlockRuleDelete(st, op, c)
default:
err = errf(KindBadRequest, "unknown operation %q", op.Kind)
}
@@ -831,6 +839,9 @@ func applyViewDelete(st *model.State, op model.Op, c *applyCtx) error {
if err := decode(op.Data, &in); err != nil {
return err
}
if users := blockingUsers(st, v.ID); len(users) > 0 {
return errf(KindConflict, "view %s is used by %s; change them to other views first", v.Name, strings.Join(users, ", "))
}
zones := st.ZonesInView(v.ID)
if len(zones) > 0 && !in.Cascade {
var names []string