diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d1224..365bd21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,35 @@ All notable changes to Jarl are documented here. The format follows ## [Unreleased] +### Added + +- Blocking (docs/blocking.md): named answers "doesn't exist" for ads, + trackers and malware, using BIND's response policy zones. + - Blocklists in Adblock, hosts or plain-domain format, downloaded by jarld + and checked daily (`blocking.refresh`). A list is downloaded only when + the server says it changed, checked with named-checkzone, and reloaded + only when its content differs; a failed update keeps the previous + version and shows on the dashboard. + - Your own block and allow rules, for a domain and the names below it or + the exact name. Rules take precedence over every list, and the most + specific rule wins. + - Lists and rules apply to chosen views, or to every view that offers + recursion. They are configuration: staged, reviewed and reverted like + any change. + - Checking a name says whether it is blocked and by which rule or list + entry, with an Allow button in the web UI. + - Web UI: a Blocking page, with well-known lists to pick and named's + memory per list. CLI: `jarl blocklist` and `jarl block`. API: + `/blocking/lists`, `/blocking/rules`, `/blocking/check`. + - Statistics: blocked queries and their share, blocked per list, most + blocked names (`jarl stats top blocked`). Blocked answers no longer + count as names that don't exist. + - A `blocking` permission; database schema 6. + +### Fixed + +- docs/validation.md lacked the `disabled_conflict` warning added in 0.3.0. + ## [0.3.0] - 2026-10-09 ### Added diff --git a/README.md b/README.md index 92b17e5..9f2b76c 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,10 @@ error: www.adbraun.lol has a CNAME, so it can't have any other records (found A) queries, record types, cache hit rate. Shown in the web UI and the CLI, and exportable as JSON. Stored as aggregates in a separate database with retention you set. +* **Blocking** like Pi-hole, done by named itself: blocklists (Adblock, + hosts or plain domains) downloaded and kept up to date, your own block + and allow rules that beat every list, per view, with "why is this + blocked?" checks and blocked counts in the statistics. * **Let's Encrypt DNS-01** built in: `jarl acme` plus certbot, acme.sh and lego hooks, an HTTPS-only endpoint for lego's and Traefik's `httpreq` provider, and CNAME delegation to a dedicated ACME zone. @@ -116,6 +120,7 @@ in `/etc/jarl/jarl.yaml` to reach it from your management network. | [docs/dnssec.md](docs/dnssec.md) | DNSSEC signing, going public with your own name servers, Porkbun | | [docs/acme.md](docs/acme.md) | Let's Encrypt DNS-01 with certbot, acme.sh, lego and Traefik | | [docs/stats.md](docs/stats.md) | Query statistics: enabling, what is collected, retention | +| [docs/blocking.md](docs/blocking.md) | Blocking ads, trackers and malware: blocklists, your own rules, views, memory | | [docs/validation.md](docs/validation.md) | Every check Jarl makes | | [docs/operations.md](docs/operations.md) | Backups, drift, failures, recovery, monitoring | | [docs/development.md](docs/development.md) | Code layout, tests against real BIND, conventions | diff --git a/cmd/jarl/blocking.go b/cmd/jarl/blocking.go new file mode 100644 index 0000000..4afe5df --- /dev/null +++ b/cmd/jarl/blocking.go @@ -0,0 +1,400 @@ +package main + +import ( + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/spf13/cobra" + + "jarl/internal/api" + "jarl/internal/blocking" +) + +// ---- blocking (docs/blocking.md) + +// viewsBody adds the --view flags to a request body: names, or "auto" for +// every view that offers recursion. +func viewsBody(body map[string]any, views []string) { + if len(views) == 1 && strings.EqualFold(views[0], "auto") { + body["views"] = []string{} + } else if len(views) > 0 { + body["views"] = views + } +} + +func viewsText(views []string, auto bool) string { + switch { + case auto: + return "auto (" + strings.Join(views, ", ") + ")" + case len(views) == 0: + return "-" + } + return strings.Join(views, ", ") +} + +func blocklistCmd() *cobra.Command { + cmd := &cobra.Command{Use: "blocklist", Aliases: []string{"blocklists"}, Short: "Blocklists: domains to block, downloaded and kept up to date", + Long: `Blocklists are downloaded by jarld and turned into response policy zones, so +named answers "doesn't exist" for the domains on them (docs/blocking.md). +Hosts, plain domain and Adblock (||domain^) lists work. Your own block and +allow rules ('jarl block') take precedence over every list.`} + var views []string + var comment string + + list := &cobra.Command{ + Use: "list", + Short: "List blocklists and their state", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + var lists []api.Blocklist + if err := c.Get(ctx(), "/blocking/lists", nil, &lists); err != nil { + return err + } + return output(lists, func(w io.Writer) { + fmt.Fprintln(w, "NAME\tSTATE\tDOMAINS\tVIEWS\tUPDATED\tMEMORY\tURL") + for _, b := range lists { + domains, updated, mem := "-", "-", "-" + if s := b.Status; s != nil && s.UpdatedAt != nil { + domains, updated = fmt.Sprint(s.Entries), s.UpdatedAt.Local().Format("2006-01-02 15:04") + } + if b.MemoryMB > 0 { + mem = fmt.Sprintf("~%d MB", b.MemoryMB) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", b.Name, b.State, domains, viewsText(b.Views, b.AutoViews), updated, mem, b.URL) + } + for _, b := range lists { + if b.Status != nil && b.Status.Error != "" { + fmt.Fprintf(w, "\n%s: %s\n", b.Name, b.Status.Error) + } + } + }) + }, + } + + add := &cobra.Command{ + Use: "add NAME URL", + Short: "Add a blocklist (jarld downloads it right away)", + Long: `Add a blocklist. NAME is a short label (letters, digits, hyphens). Without +--view the list applies to every view that offers recursion. + + jarl blocklist add hagezi-pro https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt + jarl blocklist add stevenblack https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts --view internal`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + body := map[string]any{"name": args[0], "url": args[1]} + viewsBody(body, views) + if comment != "" { + body["comment"] = comment + } + resp, err := c.Mutate(ctx(), "POST", "/blocking/lists", nil, body) + if err != nil { + return err + } + if err := report(resp); err != nil || flagQuiet || !resp.Applied { + return err + } + fmt.Println("jarld downloads the list in the background; 'jarl blocklist list' shows when it's in use.") + return nil + }, + } + add.Flags().StringArrayVar(&views, "view", nil, "a view it applies to (repeat for several; default: every view that offers recursion)") + add.Flags().StringVar(&comment, "comment", "", "comment") + + var rename, newURL string + update := &cobra.Command{ + Use: "update LIST", + Short: "Change a blocklist's name, URL, views or comment", + Example: ` jarl blocklist update hagezi --url https://example.org/new-list.txt + jarl blocklist update hagezi --view internal --view guest + jarl blocklist update hagezi --view auto # every view that offers recursion`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + body := map[string]any{} + f := cmd.Flags() + if f.Changed("name") { + body["name"] = rename + } + if f.Changed("url") { + body["url"] = newURL + } + if f.Changed("comment") { + body["comment"] = comment + } + viewsBody(body, views) + if len(body) == 0 { + return fmt.Errorf("nothing to change (see --help)") + } + resp, err := c.Mutate(ctx(), "PATCH", "/blocking/lists/"+url.PathEscape(args[0]), nil, body) + if err != nil { + return err + } + return report(resp) + }, + } + update.Flags().StringVar(&rename, "name", "", "new name") + update.Flags().StringVar(&newURL, "url", "", "new URL") + update.Flags().StringArrayVar(&views, "view", nil, `a view it applies to (repeat for several; "auto": every view that offers recursion)`) + update.Flags().StringVar(&comment, "comment", "", "new comment") + + toggle := func(disable bool) *cobra.Command { + verb := map[bool]string{true: "disable", false: "enable"}[disable] + return &cobra.Command{ + Use: verb + " LIST", + Short: map[bool]string{true: "Stop using a blocklist (it is kept, and re-enabled without downloading)", false: "Use a disabled blocklist again"}[disable], + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + resp, err := c.Mutate(ctx(), "PATCH", "/blocking/lists/"+url.PathEscape(args[0]), nil, map[string]any{"disabled": disable}) + if err != nil { + return err + } + return report(resp) + }, + } + } + + var yes bool + del := &cobra.Command{ + Use: "delete LIST", + Short: "Delete a blocklist", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if !confirm("Delete blocklist "+args[0]+"?", yes) { + return fmt.Errorf("canceled") + } + c, err := newClient() + if err != nil { + return err + } + resp, err := c.Mutate(ctx(), "DELETE", "/blocking/lists/"+url.PathEscape(args[0]), nil, nil) + if err != nil { + return err + } + return report(resp) + }, + } + del.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask") + + var force bool + refresh := &cobra.Command{ + Use: "refresh [LIST...]", + Short: "Download blocklists now (all enabled ones without LIST)", + Long: `Download blocklists now instead of waiting for their next check +(blocking.refresh in jarl.yaml, daily by default). A list is only replaced +when its content changed; --force downloads it even when the server says +it hasn't.`, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + names := args + if len(names) == 0 { + var lists []api.Blocklist + if err := c.Get(ctx(), "/blocking/lists", nil, &lists); err != nil { + return err + } + for _, b := range lists { + if b.State != "disabled" && b.State != "idle" { + names = append(names, b.Name) + } + } + } + if flagDryRun { + fmt.Printf("would download: %s\n", strings.Join(names, ", ")) + return nil + } + failed := 0 + for _, n := range names { + var b api.Blocklist + start := time.Now() + err := c.Do(ctx(), "POST", "/blocking/lists/"+url.PathEscape(n)+"/refresh", q("force", map[bool]string{true: "1"}[force]), nil, &b) + switch { + case err != nil: + failed++ + fmt.Printf("%s: %v\n", n, err) + case b.Status != nil && b.Status.Error != "": + failed++ + fmt.Printf("%s: %s (the previous version stays in use)\n", n, b.Status.Error) + case b.Status != nil: + fmt.Printf("%s: %d domains (%d lines skipped), checked in %s\n", n, b.Status.Entries, b.Status.Skipped, time.Since(start).Round(100*time.Millisecond)) + } + } + if failed > 0 { + return &checkFailed{fmt.Sprintf("%d of %d lists could not be updated", failed, len(names))} + } + return nil + }, + } + refresh.Flags().BoolVar(&force, "force", false, "download even when the server says the list hasn't changed") + + cmd.AddCommand(list, add, update, toggle(true), toggle(false), del, refresh) + return cmd +} + +func blockCmd() *cobra.Command { + cmd := &cobra.Command{Use: "block", Short: "Your own block and allow rules, and checking what is blocked", + Long: `Rules block a domain, or allow it despite the blocklists ('jarl blocklist'). +A rule covers the domain and every name below it, unless --exact. Rules +take precedence over every list, and the most specific rule wins: allow +cdn.example.com overrides block example.com for cdn.example.com and the +names below it (docs/blocking.md).`} + var views []string + var comment, view string + var exact bool + + addRule := func(action string) *cobra.Command { + c := &cobra.Command{ + Use: map[string]string{"block": "add DOMAIN", "allow": "allow DOMAIN"}[action], + Short: map[string]string{"block": "Block a domain (and the names below it)", "allow": "Allow a domain (and the names below it) that a blocklist blocks"}[action], + Example: map[string]string{ + "block": " jarl block add doubleclick.net\n jarl block add telemetry.example.com --exact --view internal", + "allow": " jarl block allow cdn.example.com\n jarl block allow login.example.com --exact", + }[action], + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + body := map[string]any{"name": args[0], "action": action} + if exact { + body["exact"] = true + } + viewsBody(body, views) + if comment != "" { + body["comment"] = comment + } + resp, err := c.Mutate(ctx(), "POST", "/blocking/rules", nil, body) + if err != nil { + return err + } + return report(resp) + }, + } + c.Flags().BoolVar(&exact, "exact", false, "only this name, not the names below it") + c.Flags().StringArrayVar(&views, "view", nil, "a view it applies to (repeat for several; default: every view that offers recursion)") + c.Flags().StringVar(&comment, "comment", "", "comment") + return c + } + + list := &cobra.Command{ + Use: "list", + Short: "List your block and allow rules", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + var rules []api.BlockRule + if err := c.Get(ctx(), "/blocking/rules", nil, &rules); err != nil { + return err + } + return output(rules, func(w io.Writer) { + fmt.Fprintln(w, "ID\tACTION\tDOMAIN\tCOVERS\tVIEWS\tCOMMENT") + for _, r := range rules { + covers := "and below" + if r.Exact { + covers = "exact" + } + fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\n", r.ID, r.Action, r.Name, covers, viewsText(r.Views, r.AutoViews), r.Comment) + } + }) + }, + } + + var id int64 + var missingOK bool + del := &cobra.Command{ + Use: "delete DOMAIN | delete --id ID", + Short: "Delete the rules for a domain (or one rule by ID)", + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + var resp *api.MutationResponse + switch { + case id > 0: + resp, err = c.Mutate(ctx(), "DELETE", fmt.Sprintf("/blocking/rules/%d", id), nil, nil) + case len(args) == 1: + qv := q("name", args[0], "view", view) + if missingOK { + qv.Set("missing_ok", "1") + } + resp, err = c.Mutate(ctx(), "DELETE", "/blocking/rules", qv, nil) + default: + return fmt.Errorf("give DOMAIN, or --id") + } + if err != nil { + return err + } + if !resp.Applied && !resp.Staged && !resp.DryRun { + fmt.Println("nothing to do") + return nil + } + return report(resp) + }, + } + del.Flags().Int64Var(&id, "id", 0, "delete the rule with this ID") + del.Flags().StringVar(&view, "view", "", "only the rules that apply in this view") + del.Flags().BoolVar(&missingOK, "missing-ok", false, "succeed when there is no such rule") + + check := &cobra.Command{ + Use: "check DOMAIN", + Short: "Show whether a name is blocked, and by which rule or list", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newClient() + if err != nil { + return err + } + var matches []blocking.Match + if err := c.Get(ctx(), "/blocking/check", q("name", args[0]), &matches); err != nil { + return err + } + return output(matches, func(w io.Writer) { + if len(matches) == 0 { + fmt.Fprintln(w, "Blocking isn't set up in any view.") + return + } + for _, m := range matches { + where := "" + if m.View != "" { + where = "view " + m.View + ":\t" + } + switch { + case m.Result == "none": + fmt.Fprintf(w, "%snot blocked\n", where) + case m.List != "": + fmt.Fprintf(w, "%sblocked by blocklist %s (entry %s)\n", where, m.List, m.Entry) + default: + fmt.Fprintf(w, "%s%s by your rule %d (entry %s)\n", where, m.Result, m.RuleID, m.Entry) + } + } + }) + }, + } + + cmd.AddCommand(addRule("block"), addRule("allow"), list, del, check) + return cmd +} diff --git a/cmd/jarl/complete.go b/cmd/jarl/complete.go index b74bcbb..a4d06a6 100644 --- a/cmd/jarl/complete.go +++ b/cmd/jarl/complete.go @@ -76,6 +76,8 @@ func argCompletion(c *cobra.Command) cobra.CompletionFunc { return append([]string{"default"}, model.ViewColors...), cobra.ShellCompDirectiveNoFileComp case p == "ACL": return lookup(cmd, "acl") + case p == "LIST": + return lookup(cmd, "blocklist") case p == "USER": return lookup(cmd, "user") case p == "KEY" && parent == "tsig": @@ -199,6 +201,14 @@ func lookup(cmd *cobra.Command, kind string) ([]cobra.Completion, cobra.ShellCom for _, v := range views { addName(v.Name, v.Comment) } + case "blocklist": + var lists []api.Blocklist + if c.Get(ctx(), "/blocking/lists", nil, &lists) != nil { + return nil, none + } + for _, b := range lists { + addName(b.Name, b.URL) + } case "acl": var acls []api.ACL if c.Get(ctx(), "/acls", nil, &acls) != nil { diff --git a/cmd/jarl/main.go b/cmd/jarl/main.go index c67fa69..413ca74 100644 --- a/cmd/jarl/main.go +++ b/cmd/jarl/main.go @@ -250,7 +250,7 @@ Exit codes: 0 ok, 1 error, 3 not authenticated or not allowed, 4 not found, root.AddCommand(c) } } - add("dns", zoneCmd(), recordCmd(), viewCmd(), aclCmd(), tsigCmd(), optionCmd(), settingsCmd(), acmeCmd(), searchCmd(), statsCmd()) + add("dns", zoneCmd(), recordCmd(), viewCmd(), aclCmd(), tsigCmd(), optionCmd(), settingsCmd(), blockCmd(), blocklistCmd(), acmeCmd(), searchCmd(), statsCmd()) add("changes", applyCmd(), changesetCmd(), historyCmd()) add("admin", statusCmd(), whoamiCmd(), userCmd(), tokenCmd(), auditCmd(), checkCmd(), resyncCmd(), configCmd()) setupCompletion(root) diff --git a/cmd/jarl/stats.go b/cmd/jarl/stats.go index d82b33d..0fe34c6 100644 --- a/cmd/jarl/stats.go +++ b/cmd/jarl/stats.go @@ -92,6 +92,9 @@ export the data.`, if s.NXDomainRatio != nil { fmt.Fprintf(w, "NXDOMAIN:\t%.1f%% of queries\n", *s.NXDomainRatio*100) } + if s.BlockedRatio != nil { + fmt.Fprintf(w, "Blocked:\t%d (%.1f%% of queries): %s\n", s.Blocked, *s.BlockedRatio*100, entryLine(s.Blocklists, s.Blocked, 6)) + } fmt.Fprintf(w, "Result codes:\t%s\n", entryLine(s.RCodes, s.Queries, 6)) fmt.Fprintf(w, "Record types:\t%s\n", entryLine(s.QTypes, s.Queries, 8)) if len(s.Answers) > 0 { @@ -103,12 +106,12 @@ export the data.`, if len(s.Zones) > 0 { fmt.Fprintf(w, "Zones:\t%s\n", strings.ReplaceAll(entryLine(s.Zones, 0, 10), stats.OtherZone+" ", "other ")) } - for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain} { + for _, k := range []string{stats.KindName, stats.KindClient, stats.KindNXDomain, stats.KindBlocked} { es := s.Tops[k] if len(es) == 0 { continue } - fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names"}[k]) + fmt.Fprintf(w, "\n%s\t\n", map[string]string{stats.KindName: "Top names", stats.KindClient: "Top clients", stats.KindNXDomain: "Top NXDOMAIN names", stats.KindBlocked: "Top blocked names"}[k]) for _, e := range es { key := e.Key if e.Name != "" { @@ -125,12 +128,13 @@ export the data.`, var limit int top := &cobra.Command{ - Use: "top [names|clients|nxdomain|servfail|refused]", - Short: "The most queried names, busiest clients, names with errors, refused clients", + Use: "top [names|clients|nxdomain|servfail|refused|blocked]", + Short: "The most queried names, busiest clients, names with errors, refused clients, blocked names", Example: ` jarl stats top names --since 7d jarl stats top clients --limit 50 jarl stats top nxdomain # misconfigured hosts and typos - jarl stats top refused # who is asking for what they may not have`, + jarl stats top refused # who is asking for what they may not have + jarl stats top blocked # what blocking stops most (docs/blocking.md)`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { kind := stats.KindName diff --git a/cmd/jarld/main.go b/cmd/jarld/main.go index c97e7fe..af29378 100644 --- a/cmd/jarld/main.go +++ b/cmd/jarld/main.go @@ -123,6 +123,10 @@ func openApp(allowMissingConfig bool) (*app, error) { NoReload: cfg.BIND.NoReload, Extras: renderExtras(cfg), }, log) + eng.Limits = engine.BlocklistLimits{ + Refresh: time.Duration(cfg.Blocking.Refresh), RetryAfter: time.Hour, Timeout: 2 * time.Minute, + MaxBytes: int64(cfg.Blocking.MaxDownloadMB) << 20, MaxEntries: cfg.Blocking.MaxEntries, + } return &app{cfg: cfg, log: log, store: st, eng: eng, owner: owner}, nil } @@ -323,6 +327,7 @@ func serveCmd() *cobra.Command { } }() go auditRetention(ctx, a) + go blocklistLoop(ctx, a) a.log.Info("jarld starting", "version", version.String(), "database", a.cfg.Database, "bind_config", a.cfg.BIND.ConfigDir) err = srv.Serve(ctx, func() { sdNotify("READY=1\nSTATUS=serving") }) sdNotify("STOPPING=1") @@ -332,6 +337,28 @@ func serveCmd() *cobra.Command { } } +// blocklistLoop keeps blocklists up to date (docs/blocking.md): every few +// minutes it downloads the lists that are due, and at once when a list +// was added or changed. +func blocklistLoop(ctx context.Context, a *app) { + select { + case <-ctx.Done(): + return + case <-time.After(10 * time.Second): // let jarld finish starting + } + t := time.NewTicker(5 * time.Minute) + defer t.Stop() + for { + a.eng.RefreshDue(ctx) + select { + case <-ctx.Done(): + return + case <-t.C: + case <-a.eng.BlocklistKicks(): + } + } +} + func auditRetention(ctx context.Context, a *app) { if a.cfg.Audit.RetentionDays <= 0 { return diff --git a/docs/api.md b/docs/api.md index 66723e8..d66c602 100644 --- a/docs/api.md +++ b/docs/api.md @@ -411,6 +411,26 @@ send the request with that changeset (`?changeset=` or | GET | `/tsig-keys/{id}` | `?secret=1` includes the secret (needs `tsig:write`, audited) | | PATCH / DELETE | `/tsig-keys/{id}` | PATCH `{"secret": "..."}` rotates | +### Blocking + +Needs `blocking:read` (`blocking:write` to change). See +[blocking.md](blocking.md). + +| Method | Path | Body / query | +|--------|------|--------------| +| GET | `/blocking/lists` | each with `views` (where it applies), `auto_views` (not chosen: every view that offers recursion), `state` (ok, waiting, error, disabled, idle), `status` (`entries`, `records`, `skipped`, `checked_at`, `updated_at`, `error`) and `memory_mb` (named's memory for it, estimated) | +| POST | `/blocking/lists` | `{"name": "hagezi-multi", "url": "https://...", "views": ["internal"], "comment": ""}` (`views` empty or absent: automatic); jarld downloads it right after | +| GET / PATCH / DELETE | `/blocking/lists/{id or name}` | PATCH any of `name`, `url`, `views`, `disabled`, `comment` | +| POST | `/blocking/lists/{id or name}/refresh` | download now and wait; `?force=1` even when the server says nothing changed. Returns the list; a failed update is in `state` and `status.error` | +| GET | `/blocking/rules` | each with `views` and `auto_views` | +| POST | `/blocking/rules` | `{"name": "doubleclick.net", "action": "block", "exact": false, "views": [], "comment": ""}` (`action` block or allow; `exact`: only the name, not the names below it) | +| PATCH / DELETE | `/blocking/rules/{id}` | | +| DELETE | `/blocking/rules` | `?name=DOMAIN[&view=V][&missing_ok=1]`: the rules for a domain | +| GET | `/blocking/check` | `?name=DOMAIN`: per view with blocking, `{view, result: blocked\|allowed\|none, entry, rule_id, list}` | + +Lists and rules are configuration like the rest: they can be staged, and +listing with `?changeset=` marks them `pending`. + ### Options and settings | Method | Path | Body / query | @@ -462,6 +482,8 @@ An **Operation** is `{"op": "...", "id": 0, "base_version": 0, "data": {...}}`: | `tsig.create` / `tsig.update` / `tsig.delete` | key | key fields | | `option.create` / `option.update` / `option.delete` | option | option fields | | `settings.update` | | settings fields | +| `blocklist.create` / `blocklist.update` / `blocklist.delete` | blocklist (update/delete) | list fields; views as `views` (names) or `view_ids` | +| `blockrule.create` / `blockrule.update` / `blockrule.delete` | rule (update/delete) | rule fields, likewise | Operations later in a batch can refer to objects created earlier in it by name (zone and view names). One request can therefore create a view, a zone @@ -471,7 +493,7 @@ in it and its records. | Method | Path | Notes | |--------|------|-------| -| GET | `/status` | named status, adoption, drift, open changesets, current problems, object counts; `expiring_tokens` (tokens that expire within 14 days or expired in the last 14: all for administrators, otherwise the caller's); `backup` (administrators: `last_at`, `last_file`, `last_size`, `last_error`, `last_error_at`, `dir`, `warn_after` in seconds, `stale`) | +| GET | `/status` | named status, adoption, drift, open changesets, current problems, object counts; `expiring_tokens` (tokens that expire within 14 days or expired in the last 14: all for administrators, otherwise the caller's); `backup` (administrators: `last_at`, `last_file`, `last_size`, `last_error`, `last_error_at`, `dir`, `warn_after` in seconds, `stale`); `blocklist_errors` (with `blocking:read`: lists whose last update failed, `{name, error, updated_at}`) | | POST | `/system/check` | validation + named-checkconf of the current configuration (`options:read`) | | POST | `/system/resync` | `{"dry_run": false, "force": false}`: rewrite every file from the database and reload (admin) | | GET | `/audit` | `?user=&action=&outcome=&object_type=&changeset=&q=&since=&until=&limit=200&before=ID` | @@ -484,11 +506,11 @@ Needs `stats:read`. When statistics are disabled, every endpoint but | Method | Path | Query | Returns | |--------|------|-------|---------| | GET | `/stats/status` | | `{enabled, query_log, statistics_channel, log_file, last_poll, last_log_line, log_lines_read, first_data, retention, warnings}` | -| GET | `/stats/summary` | `since` (90m, 24h, 7d, 2w; default 24h), `top` (default 10) | totals, `avg_qps`, `peak`, `rcodes`, `qtypes`, `views`, `zones`, `answers`, `transport`, `cache_hit_ratio`, `nxdomain_ratio`, `tops` | +| GET | `/stats/summary` | `since` (90m, 24h, 7d, 2w; default 24h), `top` (default 10) | totals, `avg_qps`, `peak`, `rcodes`, `qtypes`, `views`, `zones`, `answers`, `transport`, `cache_hit_ratio`, `nxdomain_ratio` (without blocked answers), `blocked`, `blocked_ratio`, `blocklists` (blocked per list; `rules` for your rules), `tops` | | GET | `/stats/timeline` | `since`, `step` (auto, 1m, 5m, 15m, 1h, 6h, 1d), `key` (repeatable or comma separated; `rcode:*` = every counter of a group; default `queries`) | `{step, times: [unix...], series: [{key, values}]}` | -| GET | `/stats/top` | `kind` (name, client, nxdomain, servfail, refused), `since`, `limit` (default 20) | `[{key, count}]`; client and refused entries add `name`, the host name of Jarl's record for the address (the summary's client list too) | +| GET | `/stats/top` | `kind` (name, client, nxdomain, servfail, refused, blocked), `since`, `limit` (default 20) | `[{key, count}]`; client and refused entries add `name`, the host name of Jarl's record for the address (the summary's client list too) | | GET | `/stats/heatmap` | `since` (default 28d), `key` (default queries) | `{key, from, avg[7][24], samples[7][24]}`, Monday first, local time, `-1` = no data | -| GET | `/stats/breakdown` | `group` (qtype, rcode, view, zone, answer, transport, rtt, dnssec, cache, ratelimit), `since` | `[{key, count}]` | +| GET | `/stats/breakdown` | `group` (qtype, rcode, view, zone, answer, transport, rtt, dnssec, cache, ratelimit, blocklist), `since` | `[{key, count}]` | ```sh curl -sf -H "$AUTH" "$API/stats/top?kind=nxdomain&since=7d&limit=10" | jq -r '.[] | "\(.count)\t\(.key)"' diff --git a/docs/blocking.md b/docs/blocking.md new file mode 100644 index 0000000..d185565 --- /dev/null +++ b/docs/blocking.md @@ -0,0 +1,245 @@ +# Blocking + +Jarl can make named answer "this name doesn't exist" (NXDOMAIN) for ads, +trackers and malware, like Pi-hole does, for every device that uses your +server to resolve names. It has two parts: + +- **Blocklists**: lists of domains published by others, which jarld + downloads and keeps up to date. +- **Your rules**: domains you block or allow yourself. They take precedence + over every list. + +It uses BIND's own mechanism for this, response policy zones (RPZ), so named +does the blocking itself; jarld only prepares the lists. + +- [Quick start](#quick-start) +- [Where blocking applies](#where-blocking-applies) +- [Your rules](#your-rules) +- [Blocklists](#blocklists) +- [Checking a name](#checking-a-name) +- [Statistics](#statistics) +- [Memory and performance](#memory-and-performance) +- [How it works](#how-it-works) +- [Troubleshooting](#troubleshooting) + +## Quick start + +In the web UI, open **Blocking**, pick one of the well-known lists, and +click **Add list**. Or with the CLI: + +```sh +jarl blocklist add hagezi-multi https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/multi.txt +jarl blocklist list # "ok" once downloaded, usually within seconds +jarl block check ads.doubleclick.net # blocked? by what? +jarl block allow cdn.example.com # something you need was blocked +``` + +Like every change in Jarl, adding a list or rule can be staged and reviewed +first (the web UI does this by default: apply it under **Changes**). + +## Where blocking applies + +Blocking changes the answers named gives when it resolves names for a +client (recursion). It doesn't change what the internet sees of your own +zones: other resolvers ask your server without recursion. + +**Views.** A list or rule applies to the views you choose, or by default +to every view that offers recursion. With the usual split (an `internal` +view that resolves for your network, an `external` view that only answers +for your zones) that is just `internal`. A view that doesn't offer +recursion gets nothing, and naming one gives a warning. Without views, +blocking applies to the whole server. + +**What it can't do:** + +- Devices that don't use your server for DNS aren't affected. That includes + browsers using their own DNS over HTTPS, and devices with a hard-coded + resolver (8.8.8.8). A firewall rule that sends port 53 to your server, and + your network's DNS settings, are the usual answers. +- A device that validates DNSSEC itself sees a blocked name as a + validation failure rather than "doesn't exist". It's still blocked. +- A list can contain names in your own zones. They are blocked like any + other; allow them with a rule if that's not what you want. + +## Your rules + +A rule **blocks** or **allows** a domain and every name below it, or with +`--exact` (web UI: "only this exact name") just the domain itself. + +```sh +jarl block add doubleclick.net # doubleclick.net and everything below it +jarl block add telemetry.example.com --exact # just that name +jarl block allow cdn.example.com --view internal # an exception, in one view +jarl block list +jarl block delete doubleclick.net # or: --id ID +``` + +**Precedence:** your rules come before every blocklist, and among your +rules the most specific one wins. With these rules: + +| Rule | | +|------|-| +| block `example.com` | and below | +| allow `cdn.example.com` | exact | + +`example.com` and `www.example.com` are blocked, `cdn.example.com` is +allowed, and `img.cdn.example.com` is blocked again (the allow rule was for +the exact name). An allow rule beats a blocklist entry whatever the lists +say. + +There is one rule per domain and view; a second one is refused. A rule +takes effect as soon as the change is applied. + +## Blocklists + +**Formats.** Lists are read line by line; one list may mix formats: + +| Line | Blocks | +|------|--------| +| `\|\|ads.example^` (Adblock) | the name and every name below it | +| `*.ads.example` | the name and every name below it | +| `0.0.0.0 ads.example` (hosts file, any address) | just the name | +| `ads.example` (plain domain) | just the name | + +Comments (`#`, `!`), headers and blank lines are ignored. Lines that DNS +can't act on are skipped and counted: Adblock exceptions (`@@`), cosmetic +and URL rules, addresses, invalid names, and single labels (a broken list +can't block `com`). Prefer lists in Adblock format: they say which entries +cover the names below them. + +**Well-known lists** (sizes in October 2026; they change): + +| List | URL | Domains | Memory per view | +|------|-----|--------:|----------------:| +| Hagezi Light | `https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/light.txt` | 55,000 | ~65 MB | +| Hagezi Multi Normal | `https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/multi.txt` | 159,000 | ~190 MB | +| Hagezi Pro | `https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt` | 195,000 | ~235 MB | +| OISD Small | `https://small.oisd.nl/` | 58,000 | ~70 MB | +| OISD Big | `https://big.oisd.nl/` | 241,000 | ~290 MB | +| StevenBlack hosts | `https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts` | 73,000 | ~45 MB | + +One good list is usually enough; lists overlap a lot. Hagezi Multi Normal +is a balanced start; Light and OISD Small block less but almost never +something you need. + +**Updates.** jarld checks each list every `blocking.refresh` (a day by +default) and downloads it only when the server says it changed. A new +version is checked with `named-checkzone` and replaces the old one only +when its content differs; named then reloads it. If a download or check +fails, named keeps using the previous version, jarld tries again within +the hour, and the dashboard shows the error. To update now: + +```sh +jarl blocklist refresh # every list; or: refresh hagezi-multi [--force] +``` + +**Managing lists:** + +```sh +jarl blocklist update hagezi-multi --view internal # --view auto: every view with recursion +jarl blocklist update hagezi-multi --url URL --name NEW-NAME +jarl blocklist disable hagezi-multi # stop using it, keep it (enable: back without a download) +jarl blocklist delete hagezi-multi +``` + +A list's state is **ok** (in use), **waiting** (not downloaded yet), +**error** (the last update failed; the previous version stays in use), +**disabled**, or **idle** (it applies to no view, so it does nothing). + +## Checking a name + +`jarl block check NAME`, or the box at the top of the Blocking page, says +for each view whether a name is blocked or allowed, and by which rule or +list entry: + +``` +$ jarl block check img.ads.example.com +view internal: blocked by blocklist hagezi-multi (entry *.ads.example.com) +view guest: allowed by your rule 12 (entry *.ads.example.com) +``` + +Views without blocking aren't listed. + +In the web UI a blocked name gets an **Allow** button, and the statistics' +most blocked names link to this check. + +## Statistics + +With [query statistics](stats.md) on (including `query_log`), Jarl counts +what blocking does: blocked queries over time and as a share of all +queries, which rules and lists blocked them, and the most blocked names +(`jarl stats summary`, `jarl stats top blocked`, `jarl stats breakdown +blocklist`). Blocked answers are left out of the NXDOMAIN figures, which +are about names that really don't exist. + +Without statistics named doesn't log what it blocks at all. + +## Memory and performance + +**Memory** is the cost to watch. named keeps each list in memory, once per +view it applies to: about 0.6 KB per policy record (an Adblock entry is +two records: the name, and the names below it). The Blocking page and +`jarl blocklist list` show the estimate per list. Hagezi Pro in one view +took named from 25 MB to 260 MB in a test with BIND 9.20; while a list +reloads, the old and new versions briefly coexist. + +**Answers aren't slowed**: named looks blocked names up in memory, and +answers them without asking the internet. + +**Updating a list** (Hagezi Pro: download, parse, check) takes jarld about +2 seconds. While named loads the new version, the queries that the loading +thread would have answered wait, up to half a second; this only happens +when the list changed, at most once per `blocking.refresh`. + +**Other changes are unaffected**: the lists' contents never go through +Jarl's apply, so adding a record takes as long with a list as without. + +Downloads are limited by `blocking.max_download_mb` and +`blocking.max_entries` ([configuration.md](configuration.md#blocking)). + +## How it works + +For each view (or the server, without views) Jarl writes a +`response-policy` statement and the policy zones it lists, in this order: + +``` +response-policy { + zone "rules.jarl.invalid" min-update-interval 0; # your rules + zone "hagezi-multi.blocklist.jarl.invalid" min-update-interval 0; +} qname-wait-recurse no break-dnssec yes; +``` + +- Policy zones are named under `.invalid`, so they can't clash with real + zones (zone names under `jarl.invalid` are refused). They allow no + queries or transfers. +- The rules zone of each view is a file next to your zone files + (`/var/lib/bind/jarl/VIEW/rules.jarl.invalid.rpz`) and shows up in the + changeset diff like any other. +- A list's file is `/var/cache/bind/jarl/blocklists/list-ID.rpz`, shared by + the views that use it. Until the first download it is an empty policy + zone, so named always starts. Files of deleted lists are removed. +- `qname-wait-recurse no` blocks without first resolving the real answer; + `break-dnssec yes` also blocks for clients that ask for DNSSEC data; + `min-update-interval 0` applies a new version at once. +- named looks names up in a policy zone like in any zone, so a wildcard + doesn't apply below a name the zone also contains. Jarl writes the zones + so that the most specific rule still wins. +- A `response-policy` option of your own can't be combined with Jarl's + blocking in the same view: Jarl refuses the change and says so. +- Rewrites are logged to Jarl's response log when statistics read it, and + not at all otherwise. + +Lists and rules need the `blocking` permission ([permissions.md](permissions.md)). + +## Troubleshooting + +- **"no domains found in N lines"**: the URL didn't return a list (often an + HTML error page). Check it in a browser. +- **A list has no effect / is "idle"**: no view it applies to offers + recursion. Pick the view your devices use (`--view internal`). +- **Something is still answered**: `jarl block check NAME` shows whether + Jarl blocks it. If it does, the device probably doesn't use your server + (DNS over HTTPS, another resolver), or has the answer cached for a + while. +- **Something you need is blocked**: `jarl block check NAME` shows which + list; `jarl block allow NAME` overrides it. diff --git a/docs/cli.md b/docs/cli.md index 9a61d95..b607510 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -419,15 +419,33 @@ working. ```sh jarl stats status jarl stats summary --since 7d -jarl stats top names | clients | nxdomain | servfail | refused [--since 24h] [--limit 20] +jarl stats top names | clients | nxdomain | servfail | refused | blocked [--since 24h] [--limit 20] jarl stats timeline --since 6h [--key rcode:NXDOMAIN] [--table] jarl stats heatmap --since 28d -jarl stats breakdown qtype | rcode | view | zone | answer | rtt +jarl stats breakdown qtype | rcode | view | zone | answer | rtt | blocklist ``` Statistics must be enabled in `jarl.yaml`, and `-o json` exports the data. See [stats.md](stats.md). +## Blocking + +```sh +jarl blocklist list +jarl blocklist add NAME URL [--view V]... # default: every view that offers recursion +jarl blocklist update NAME [--url U] [--name N] [--view V | --view auto] [--comment C] +jarl blocklist disable | enable | delete NAME +jarl blocklist refresh [NAME...] [--force] # download now (all lists without NAME) + +jarl block add DOMAIN [--exact] [--view V]... # block it (and the names below it) +jarl block allow DOMAIN [--exact] [--view V]... # an exception that beats every list +jarl block list +jarl block delete DOMAIN [--view V] | --id ID +jarl block check DOMAIN # blocked? by which rule or list? +``` + +See [blocking.md](blocking.md). + ## ACME DNS-01 ```sh diff --git a/docs/configuration.md b/docs/configuration.md index f3c9ab6..8a0f48a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,6 +134,17 @@ jarl.yaml; chmod 640 jarl.yaml`, as the Debian package installs it); jarld warns at start when everyone can read it. At porkbun.com, switch on "API Access" for the domain, and limit the key to the domain if you can. +### Blocking + +How jarld downloads blocklists ([blocking.md](blocking.md)). The lists and +rules themselves are configuration in the database. + +| Key | Default | Meaning | +|-----|---------|---------| +| `blocking.refresh` | `24h` | How often each blocklist is checked for changes (at least `1h`); it is downloaded only when it changed ([blocking.md](blocking.md)). | +| `blocking.max_download_mb` | `64` | Larger downloads are refused (the previous version stays in use). | +| `blocking.max_entries` | `1000000` | Lists with more domains are refused. | + ## Example: management network with TLS ```yaml diff --git a/docs/development.md b/docs/development.md index a2d00a7..2236ca9 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,6 +28,8 @@ internal/ nscheck/ DNS queries to other name servers: zone serials, the parent's delegation and DS records dnssec/ reads named's DNSSEC key and state files; DS records; the signing phases (docs/dnssec.md) porkbun/ Porkbun API client (name servers, glue, DS records) and an in-memory fake for tests + blocking/ blocklist parser, response policy zone writers, and matching a name the way named does + (the downloads are in engine/blocklists.go) config/ jarld's YAML configuration deploy/ systemd units (jarld, the daily backup service and timer), sysusers.d and tmpfiles.d files deploy/debian/ the Debian package's control template and postinst/prerm/postrm diff --git a/docs/permissions.md b/docs/permissions.md index b7dff98..f20e351 100644 --- a/docs/permissions.md +++ b/docs/permissions.md @@ -135,6 +135,7 @@ RESOURCE:ACCESS [zones=PATTERNS] [views=PATTERNS] [names=PATTERNS] [types=TYPES] | `options` | server options, top-level statements, the defaults for new zones, `jarl check` | | `audit` | everyone's audit events and changesets (without it, users see only their own) | | `stats` | query statistics, including client addresses and the names they look up (read only) | +| `blocking` | blocklists and block/allow rules ([blocking.md](blocking.md)); `write` also downloads lists now | | `*` | all of the above | `ACCESS` is `read` or `write` (write includes read). Filters narrow a grant. @@ -168,7 +169,7 @@ you can then edit. | Role | Grants | |------|--------| | `readonly` | `*:read` (includes statistics) | -| `dns-editor` | `zones:write`, `records:write`, `views:read`, `acls:read`, `options:read` | +| `dns-editor` | `zones:write`, `records:write`, `views:read`, `acls:read`, `options:read`, `blocking:read` | | `operator` | `*:write` (everything except user management) | | `acme` | `records:write names=_acme-challenge.* types=TXT` | diff --git a/docs/stats.md b/docs/stats.md index ca2dae9..7e0aebb 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -102,10 +102,14 @@ lists, views, zones). named's own counters can't be filtered. | `view:NAME` | response log | answers per view (`_default` without views) | | `zone:NAME` | response log | answers per local zone; `zone:-` for names in none of your zones | | `responses` | response log | lines read (a health check) | +| `blocked`, `blocklist:NAME` | response log | queries [blocking](blocking.md) answered, and by which list (`blocklist:rules` for your own rules) | **Top lists** (per hour and per day, the `top_n` largest): `name` (most queried names), `client`, `nxdomain` (names answered NXDOMAIN), `servfail` -(names answered SERVFAIL), `refused` (clients that were refused). +(names answered SERVFAIL), `refused` (clients that were refused), `blocked` +(names blocking answered). Blocked answers are NXDOMAIN too, but they +aren't counted as names that don't exist: not in the `nxdomain` list, and +not in the summary's NXDOMAIN ratio. Nothing else is stored. In particular there's no per-query record, so you can't look up "who asked for X at 14:02". That is deliberate, both for size diff --git a/docs/validation.md b/docs/validation.md index 2da5625..23efa1b 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -30,6 +30,7 @@ showing them. | `record_out_of_zone` | error | The name must be inside the zone (catches `www.example.com` without the final dot) | | `record_duplicate` | error | The same record can't exist twice | | `cname_conflict` | error | A CNAME must be the only record at its name (RFC 1034), so no CNAME next to A, MX, TXT... | +| `disabled_conflict` | warning | A disabled record that couldn't be enabled again as the zone is now (a CNAME now shares its name) | | `cname_multiple` | error | A name can have only one CNAME | | `cname_apex` | error | No CNAME at the zone apex (it needs SOA and NS) | | `cname_loop` | error | CNAMEs that point back to themselves | @@ -100,6 +101,19 @@ applied. | `dnssec_old_keys` | info | Signing a zone whose key directory still has old keys | | `serial_not_higher` | error | An explicit serial on a signed zone must be above the one named has loaded (automatic serials are raised by themselves) | +## Blocking + +From [blocking.md](blocking.md). + +| Code | Severity | Rule | +|------|----------|------| +| `blocking_rule_duplicate` | error | One rule per domain and view | +| `blocking_response_policy` | error | No `response-policy` option where Jarl's blocking writes that statement (the view, or the server) | +| `zone_name_reserved` | error | Zone names under `jarl.invalid` are reserved for blocking's policy zones | +| `blocking_view_missing` | error | A list or rule names a view that doesn't exist | +| `blocking_no_view` | warning | A list or rule applies to no view: none offers recursion, and blocking only changes recursive answers | +| `blocking_no_recursion` | warning | A list or rule names a view that doesn't offer recursion, where it has no effect | + ## Security lints | Code | Severity | What it catches | diff --git a/internal/api/types.go b/internal/api/types.go index 82a3aad..c148542 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -242,6 +242,17 @@ type StatusResponse struct { Backup *BackupStatus `json:"backup,omitempty"` // TOTP is about two-factor login across users (administrators only). TOTP *TOTPOverview `json:"totp,omitempty"` + // BlocklistErrors are blocklists whose last update failed (for those + // who may see blocking). + BlocklistErrors []BlocklistError `json:"blocklist_errors,omitempty"` +} + +// BlocklistError is a blocklist whose last update failed. UpdatedAt is +// the version named keeps using (nil: none yet). +type BlocklistError struct { + Name string `json:"name"` + Error string `json:"error"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` } // TOTPOverview is what the dashboard reports about two-factor login. @@ -471,3 +482,28 @@ type RegistrarChangeResponse struct { Checks []nscheck.Finding `json:"checks"` Warnings []string `json:"warnings"` } + +// Blocklist is a blocklist as returned by the API (docs/blocking.md). +type Blocklist struct { + model.Blocklist + // Views are the views it applies to; AutoViews is set when that is + // "every view that offers recursion" (view_ids is empty). + Views []string `json:"views"` + AutoViews bool `json:"auto_views,omitempty"` + // State is ok, waiting (not downloaded yet), error (the last update + // failed; the previous version stays in use), disabled, or idle (it + // applies to no view). + State string `json:"state"` + Status *model.BlocklistStatus `json:"status,omitempty"` + // MemoryMB estimates what named needs for the list, in all its views. + MemoryMB int `json:"memory_mb,omitempty"` + Pending string `json:"pending,omitempty"` +} + +// BlockRule is a block or allow rule as returned by the API. +type BlockRule struct { + model.BlockRule + Views []string `json:"views"` + AutoViews bool `json:"auto_views,omitempty"` + Pending string `json:"pending,omitempty"` +} diff --git a/internal/auth/grants.go b/internal/auth/grants.go index 12df0f1..794deec 100644 --- a/internal/auth/grants.go +++ b/internal/auth/grants.go @@ -12,19 +12,20 @@ import ( // Resources that grants apply to. const ( - ResZones = "zones" // zone definitions and zone options - ResRecords = "records" // records in zones - ResViews = "views" // views and view options - ResACLs = "acls" - ResTSIG = "tsig" // TSIG keys - ResOptions = "options" // server options, top-level statements, settings - ResAudit = "audit" // audit log and everyone's changesets - ResStats = "stats" // query statistics - ResAll = "*" + ResZones = "zones" // zone definitions and zone options + ResRecords = "records" // records in zones + ResViews = "views" // views and view options + ResACLs = "acls" + ResTSIG = "tsig" // TSIG keys + ResOptions = "options" // server options, top-level statements, settings + ResAudit = "audit" // audit log and everyone's changesets + ResStats = "stats" // query statistics + ResBlocking = "blocking" // blocklists and block/allow rules + ResAll = "*" ) // Resources lists the resources in display order. -var Resources = []string{ResZones, ResRecords, ResViews, ResACLs, ResTSIG, ResOptions, ResAudit, ResStats} +var Resources = []string{ResZones, ResRecords, ResViews, ResACLs, ResTSIG, ResOptions, ResAudit, ResStats, ResBlocking} // Access levels. const ( @@ -377,6 +378,7 @@ var RoleTemplates = map[string][]model.Grant{ {Resource: ResViews, Access: Read}, {Resource: ResACLs, Access: Read}, {Resource: ResOptions, Access: Read}, + {Resource: ResBlocking, Access: Read}, }, "operator": {{Resource: ResAll, Access: Write}}, "acme": {{Resource: ResRecords, Access: Write, Names: []string{"_acme-challenge.*"}, Types: []string{"TXT"}}}, diff --git a/internal/blocking/blocking_test.go b/internal/blocking/blocking_test.go new file mode 100644 index 0000000..d4f941b --- /dev/null +++ b/internal/blocking/blocking_test.go @@ -0,0 +1,180 @@ +package blocking + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParse(t *testing.T) { + src := "\uFEFF[Adblock Plus]\r\n" + `! Title: test list +# a comment + +||ads.example^ +||Tracker.Example.^$important +||cosmetic.example^$third-party +@@||allowed.example^ +example.org##.banner +|https://url.example/path +0.0.0.0 hosts.example other.example # two names +127.0.0.1 localhost +127.0.0.1 localhost.localdomain +::1 ip6-localhost +0.0.0.0 0.0.0.0 +plain.example +*.wild.example +single +1.2.3.4 +bad_label!.example +münchen.example +www.ads.example +` + strings.Repeat("a", 64) + ".example\n" + l, err := Parse(strings.NewReader(src), Limits{}) + if err != nil { + t.Fatal(err) + } + want := map[string]uint8{ + "ads.example": Exact | Below, "tracker.example": Exact | Below, + "hosts.example": Exact, "other.example": Exact, "plain.example": Exact, + "wild.example": Exact | Below, "xn--mnchen-3ya.example": Exact, "www.ads.example": Exact, + } + if len(l.Domains) != len(want) { + t.Errorf("domains = %v", l.Domains) + } + for n, f := range want { + if l.Domains[n] != f { + t.Errorf("%s = %d, want %d", n, l.Domains[n], f) + } + } + // Skipped: $third-party, @@, ##, |url, localhost (x3), 0.0.0.0, single, 1.2.3.4, bad_label!, 64-char label. + if l.Skipped != 12 || l.Lines != 19 { + t.Errorf("lines %d, skipped %d", l.Lines, l.Skipped) + } + + Compact(l) + if _, ok := l.Domains["www.ads.example"]; ok { + t.Error("www.ads.example is covered by ||ads.example^ and should be dropped") + } + var sb strings.Builder + n, hash, err := FeedZone(&sb, "test", 7, l) + if err != nil || n != 10 { + t.Fatalf("FeedZone: %d records, %v", n, err) + } + for _, line := range []string{"ads.example\tCNAME\t.\n", "*.ads.example\tCNAME\t.\n", "plain.example\tCNAME\t.\n", "@\tSOA\tlocalhost. hostmaster.localhost. 7 "} { + if !strings.Contains(sb.String(), line) { + t.Errorf("zone lacks %q:\n%s", line, sb.String()) + } + } + if strings.Contains(sb.String(), "*.plain.example") || strings.Contains(sb.String(), "$ORIGIN") { + t.Errorf("zone:\n%s", sb.String()) + } + // The hash ignores the serial. + var other strings.Builder + if _, h2, _ := FeedZone(&other, "test", 8, l); h2 != hash { + t.Error("hash depends on the serial") + } +} + +func TestParseLimits(t *testing.T) { + _, err := Parse(strings.NewReader("a.example\nb.example\na.example\nc.example\n"), Limits{MaxEntries: 2}) + if !errors.Is(err, ErrTooManyEntries) { + t.Errorf("err = %v", err) + } + if _, err := Parse(strings.NewReader("a.example\nb.example\na.example\n"), Limits{MaxEntries: 2}); err != nil { + t.Errorf("a repeated name counted against the limit: %v", err) + } + l, _ := Parse(strings.NewReader("long-name.example\nshort.example\n"), Limits{MaxNameLen: 15}) + if len(l.Domains) != 1 || l.Skipped != 1 { + t.Errorf("max name length: %v", l.Domains) + } +} + +func TestRuleEntries(t *testing.T) { + rules := []Rule{ + {Name: "ads.example", Action: ActionBlock}, // and everything below + {Name: "cdn.ads.example", Action: ActionAllow, Exact: true}, // just this name + {Name: "wide.example", Action: ActionBlock}, // and below + {Name: "ok.wide.example", Action: ActionAllow}, // and below + {Name: "deep.x.y.wide.example", Action: ActionAllow, Exact: true}, // two empty names in between + {Name: "solo.example", Action: ActionBlock, Exact: true}, + } + e := RuleEntries(rules) + for name, want := range map[string]string{ + "ads.example": ActionBlock, + "x.ads.example": ActionBlock, + "cdn.ads.example": ActionAllow, + "y.cdn.ads.example": ActionBlock, // the wildcard above still applies + "ok.wide.example": ActionAllow, + "a.ok.wide.example": ActionAllow, + "y.wide.example": ActionBlock, // exists only because of a deeper rule + "x.y.wide.example": ActionBlock, + "z.x.y.wide.example": ActionBlock, + "deep.x.y.wide.example": ActionAllow, + "below.deep.x.y.wide.example": ActionBlock, + "solo.example": ActionBlock, + "other.example": "", + "sub.solo.example": "", + "example": "", + "unrelated.org": "", + } { + _, act, ok := MatchRules(e, name) + if !ok { + act = "" + } + if act != want { + t.Errorf("%s: %q, want %q", name, act, want) + } + } + z := RulesZoneFile("rules", rules) + if !strings.Contains(z, "cdn.ads.example\tCNAME\trpz-passthru.\n*.cdn.ads.example\tCNAME\t.\n") { + t.Errorf("rules zone:\n%s", z) + } + if strings.Index(z, "ads.example\t") > strings.Index(z, "*.ads.example\t") { + t.Errorf("a name should come before its wildcard:\n%s", z) + } +} + +func TestMatchFeed(t *testing.T) { + l, _ := Parse(strings.NewReader("||tracker.example^\nexact.example\n||deep.a.example^\n"), Limits{}) + Compact(l) + path := filepath.Join(t.TempDir(), "list.rpz") + f, _ := os.Create(path) + FeedZone(f, "t", 1, l) + f.Close() + for name, want := range map[string]string{ + "tracker.example": "tracker.example", + "a.b.tracker.example": "*.tracker.example", + "exact.example": "exact.example", + "sub.exact.example": "", + "x.deep.a.example": "*.deep.a.example", + "a.example": "", + "example": "", + } { + owner, ok, err := MatchFeed(path, name) + if err != nil { + t.Fatal(err) + } + if !ok { + owner = "" + } + if owner != want { + t.Errorf("%s: %q, want %q", name, owner, want) + } + } +} + +func TestNames(t *testing.T) { + if ListZone("hagezi") != "hagezi.blocklist.jarl.invalid" || ListName("hagezi.blocklist.jarl.invalid.") != "hagezi" || ListName(RulesZone) != "" || ListName("x.y.blocklist.jarl.invalid") != "" { + t.Error("zone names") + } + if n, err := NormalizeName("Ads.Example.COM."); err != nil || n != "ads.example.com" { + t.Errorf("NormalizeName = %q, %v", n, err) + } + for _, bad := range []string{"com", "1.2.3.4", "a b.example", "*.example.com", ""} { + if _, err := NormalizeName(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} diff --git a/internal/blocking/parse.go b/internal/blocking/parse.go new file mode 100644 index 0000000..10221ca --- /dev/null +++ b/internal/blocking/parse.go @@ -0,0 +1,198 @@ +// Package blocking turns blocklists and block/allow rules into BIND +// response policy zones (RPZ), and answers which rule or list matches a +// name. +// +// A policy zone holds one rule per owner name: "ads.example CNAME ." +// answers NXDOMAIN for ads.example, "*.ads.example CNAME ." for every name +// below it, and "CNAME rpz-passthru." answers normally (an exception). +// named looks names up in a policy zone like in any zone, so wildcards +// follow DNS rules: a wildcard doesn't apply below a name that exists in +// the zone. The writers here add the rules that keep "the most specific +// rule wins" true despite that (see RulesZone). +package blocking + +import ( + "bufio" + "errors" + "fmt" + "io" + "net/netip" + "strings" + "unicode/utf8" + + "jarl/internal/dnsname" +) + +// Flags say what a list entry covers. +const ( + Exact uint8 = 1 << iota // the name itself + Below // every name below it +) + +// List is a parsed blocklist. +type List struct { + // Domains maps names (lowercase, without the final dot) to flags. + Domains map[string]uint8 + // Lines is the number of lines with an entry (not comments or blank + // lines); Skipped how many of them were not usable. + Lines, Skipped int +} + +// Limits bound what Parse accepts. +type Limits struct { + MaxEntries int // distinct domains (0: no limit) + MaxNameLen int // longest name in characters (0: 253) +} + +// ErrTooManyEntries is returned when a list has more than MaxEntries +// domains. +var ErrTooManyEntries = errors.New("too many entries") + +// Parse reads a blocklist. Each line is one of: +// +// ||ads.example^ Adblock style: the name and everything below it +// *.ads.example the name and everything below it +// 0.0.0.0 ads.example hosts file (any address; several names allowed) +// ads.example just this name +// +// Comments (#, !), Adblock headers ([Adblock Plus]) and blank lines are +// ignored. Lines that can't be used for DNS blocking (exceptions, +// cosmetic and URL rules, addresses, single-label names, invalid names) +// are counted as skipped. +func Parse(r io.Reader, lim Limits) (*List, error) { + maxLen := lim.MaxNameLen + if maxLen <= 0 || maxLen > 253 { + maxLen = 253 + } + l := &List{Domains: map[string]uint8{}} + full := false // a new name was refused because of MaxEntries + add := func(name string, flags uint8) bool { + name, ok := normalize(name, maxLen) + if !ok { + return false + } + if _, seen := l.Domains[name]; !seen && lim.MaxEntries > 0 && len(l.Domains) >= lim.MaxEntries { + full = true + return false + } + l.Domains[name] |= flags + return true + } + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + first := true + for sc.Scan() { + line := sc.Text() + if first { + line = strings.TrimPrefix(line, "\uFEFF") + first = false + } + line = strings.TrimSpace(line) + if line == "" || line[0] == '#' || line[0] == '!' || line[0] == '[' { + continue + } + l.Lines++ + ok := false + switch { + case strings.HasPrefix(line, "||"): + ok = add(adblockDomain(line[2:]), Exact|Below) + case strings.ContainsAny(line, "|/$^@") || strings.Contains(line, "##"): + // Exceptions (@@), URL and cosmetic rules: not for DNS. + default: + if i := strings.Index(line, "#"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + fields := strings.Fields(line) + switch { + case len(fields) == 1 && strings.HasPrefix(fields[0], "*."): + ok = add(fields[0][2:], Exact|Below) + case len(fields) == 1: + ok = add(fields[0], Exact) + case len(fields) > 1 && isAddr(fields[0]): + for _, f := range fields[1:] { + if add(f, Exact) { + ok = true + } + } + } + } + if full { + return l, fmt.Errorf("%w: the list has more than %d domains", ErrTooManyEntries, lim.MaxEntries) + } + if !ok { + l.Skipped++ + } + } + if err := sc.Err(); err != nil { + return l, err + } + return l, nil +} + +// adblockDomain returns the domain of an Adblock rule's body +// ("ads.example^" or "ads.example^$important"), or "" when the rule +// does more than block a domain (other modifiers, paths, wildcards). +func adblockDomain(s string) string { + d, rest, _ := strings.Cut(s, "^") + if rest != "" && rest != "$important" { + return "" + } + if strings.ContainsAny(d, "*/:|$") { + return "" + } + return d +} + +func isAddr(s string) bool { + _, err := netip.ParseAddr(s) + return err == nil +} + +// localNames are hosts-file names that aren't blocklist entries. +var localNames = map[string]bool{"localhost.localdomain": true, "ip6-localhost": true, "ip6-loopback": true} + +// normalize lowercases a name, removes the final dot, converts +// international names to their ASCII form, and checks it is usable: at +// least two labels of letters, digits, hyphens and underscores, and not an +// address. +func normalize(name string, maxLen int) (string, bool) { + name = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), ".") + if name == "" || localNames[name] || isAddr(name) { + return "", false + } + for i := 0; i < len(name); i++ { + if name[i] >= utf8.RuneSelf { + a, err := dnsname.ToASCII(name) + if err != nil { + return "", false + } + name = strings.TrimSuffix(strings.ToLower(a), ".") + break + } + } + if len(name) > maxLen || !strings.Contains(name, ".") { + return "", false + } + for _, label := range strings.Split(name, ".") { + if label == "" || len(label) > 63 { + return "", false + } + for i := 0; i < len(label); i++ { + c := label[i] + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_') { + return "", false + } + } + } + return name, true +} + +// NormalizeName checks and normalizes a domain the way list entries are +// (for rules entered by hand). The name must have at least two labels. +func NormalizeName(name string) (string, error) { + n, ok := normalize(name, 253-len(RulesZone)-2) + if !ok { + return "", fmt.Errorf("%q is not a domain name that can be blocked (it needs at least two labels of letters, digits, hyphens or underscores, like ads.example.com)", name) + } + return n, nil +} diff --git a/internal/blocking/zone.go b/internal/blocking/zone.go new file mode 100644 index 0000000..8f8b55a --- /dev/null +++ b/internal/blocking/zone.go @@ -0,0 +1,306 @@ +package blocking + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "sort" + "strings" +) + +// Policy zone names. They are under .invalid (RFC 6761), so they can +// never clash with a real zone, and named only uses them internally. +const ( + // RulesZone holds the rules entered in Jarl (one copy per view). + RulesZone = "rules.jarl.invalid" + // listSuffix follows a list's name: hagezi.blocklist.jarl.invalid. + listSuffix = ".blocklist.jarl.invalid" +) + +// ListZone returns the policy zone name of a blocklist. +func ListZone(name string) string { return name + listSuffix } + +// ListName returns the blocklist name of a policy zone ("" for the rules +// zone and zones that aren't Jarl's). +func ListName(zone string) string { + zone = strings.TrimSuffix(strings.ToLower(zone), ".") + if n, ok := strings.CutSuffix(zone, listSuffix); ok && !strings.Contains(n, ".") { + return n + } + return "" +} + +// Match says what blocking does with a name in one view. +type Match struct { + View string `json:"view,omitempty"` + Result string `json:"result"` // blocked, allowed or none + Entry string `json:"entry,omitempty"` // the matching entry: the name, or *.parent + RuleID int64 `json:"rule_id,omitempty"` + List string `json:"list,omitempty"` // the blocklist, when a list matched +} + +// Actions of rules. +const ( + ActionBlock = "block" + ActionAllow = "allow" +) + +// cname is the CNAME target of each action: "." answers NXDOMAIN, +// rpz-passthru. answers as if there were no rule. +var cname = map[string]string{ActionBlock: ".", ActionAllow: "rpz-passthru."} + +// header starts a policy zone file. There is no $ORIGIN: names are +// relative to the zone, so a list can be renamed without rewriting its +// file. The short TTLs make a change reach clients' caches quickly. +func header(title string, serial uint32) string { + return fmt.Sprintf("; %s\n; Generated by Jarl. DO NOT EDIT.\n$TTL 60\n@\tSOA\tlocalhost. hostmaster.localhost. %d 3600 600 86400 60\n\tNS\tlocalhost.\n", title, serial) +} + +// Placeholder is the content of a list's zone file before the list has +// been downloaded: a valid, empty policy zone. +func Placeholder() string { + return header("Blocklist not downloaded yet.", 1) +} + +// Rule is a block or allow rule entered by hand. +type Rule struct { + Name string // lowercase, without the final dot + Action string // ActionBlock or ActionAllow + Exact bool // only the name itself, not the names below it +} + +// RuleEntries returns the policy zone owners for rules (relative names, +// "*." for wildcards) and their actions, with the entries that make the +// most specific rule win added: named won't apply a wildcard below a name +// that exists in the zone, so every name below a wildcard that exists +// (because a more specific rule mentions it) gets the wildcard's action +// for itself and for the names below it, unless a rule says otherwise. +// Callers make sure no two rules have the same name and scope. +func RuleEntries(rules []Rule) map[string]string { + e := map[string]string{} + for _, r := range rules { + e[r.Name] = r.Action + if !r.Exact { + e["*."+r.Name] = r.Action + } + } + // Every name that exists in the zone: owners and their ancestors. + nodes := map[string]bool{} + for owner := range e { + n := strings.TrimPrefix(owner, "*.") + for { + nodes[n] = true + i := strings.IndexByte(n, '.') + if i < 0 { + break + } + n = n[i+1:] + } + } + ordered := make([]string, 0, len(nodes)) + for n := range nodes { + ordered = append(ordered, n) + } + // Shallow names first, so a filled-in wildcard is inherited further + // down. + sort.Slice(ordered, func(i, j int) bool { + di, dj := strings.Count(ordered[i], "."), strings.Count(ordered[j], ".") + if di != dj { + return di < dj + } + return ordered[i] < ordered[j] + }) + for _, n := range ordered { + inherited := "" + for a := n; ; { + i := strings.IndexByte(a, '.') + if i < 0 { + break + } + a = a[i+1:] + if act, ok := e["*."+a]; ok { + inherited = act + break + } + } + if inherited == "" { + continue + } + if _, ok := e[n]; !ok { + e[n] = inherited + } + if _, ok := e["*."+n]; !ok { + e["*."+n] = inherited + } + } + return e +} + +// RulesZoneFile renders the rules zone of a view. +func RulesZoneFile(title string, rules []Rule) string { + e := RuleEntries(rules) + owners := make([]string, 0, len(e)) + for o := range e { + owners = append(owners, o) + } + sort.Slice(owners, func(i, j int) bool { return ownerLess(owners[i], owners[j]) }) + var sb strings.Builder + sb.WriteString(header(title, 1)) + for _, o := range owners { + fmt.Fprintf(&sb, "%s\tCNAME\t%s\n", o, cname[e[o]]) + } + return sb.String() +} + +// ownerLess sorts names by their labels from the right (example, +// ads.example, *.ads.example, cdn.ads.example), so related rules are next +// to each other. +func ownerLess(a, b string) bool { + ka, kb := reverseLabels(a), reverseLabels(b) + return ka < kb +} + +func reverseLabels(n string) string { + labels := strings.Split(n, ".") + for i, j := 0, len(labels)-1; i < j; i, j = i+1, j-1 { + labels[i], labels[j] = labels[j], labels[i] + } + // A wildcard sorts right after the name it belongs to. + return strings.ReplaceAll(strings.Join(labels, "\x00"), "*", "\x01") +} + +// MatchRules finds the rule entry named would apply to name (lowercase, +// without the final dot) in a rules zone with entries e: the name itself, +// or else the wildcard of its closest existing ancestor. ok is false when +// no entry applies. +func MatchRules(e map[string]string, name string) (owner, action string, ok bool) { + if act, found := e[name]; found { + return name, act, true + } + nodes := map[string]bool{} + for o := range e { + n := strings.TrimPrefix(o, "*.") + for { + nodes[n] = true + i := strings.IndexByte(n, '.') + if i < 0 { + break + } + n = n[i+1:] + } + } + if nodes[name] { + return "", "", false // the name exists without a rule of its own + } + for a := name; ; { + i := strings.IndexByte(a, '.') + if i < 0 { + return "", "", false + } + a = a[i+1:] + if nodes[a] { + if act, found := e["*."+a]; found { + return "*." + a, act, true + } + return "", "", false + } + } +} + +// Compact drops entries another entry already covers (a name below a +// name whose subdomains are blocked). That keeps the zone small, and it +// matters for correctness: no name exists below a wildcard, so every +// wildcard applies to all names below it. +func Compact(l *List) { + for n := range l.Domains { + for a := n; ; { + i := strings.IndexByte(a, '.') + if i < 0 { + break + } + a = a[i+1:] + if l.Domains[a]&Below != 0 { + delete(l.Domains, n) + break + } + } + } +} + +// FeedZone writes a list's policy zone to w and returns the number of +// records and a hash of them (the serial isn't part of the hash, so it +// tells whether the content changed). l must be compacted. +func FeedZone(w io.Writer, title string, serial uint32, l *List) (records int, hash string, err error) { + names := make([]string, 0, len(l.Domains)) + for n := range l.Domains { + names = append(names, n) + } + sort.Strings(names) + bw := bufio.NewWriterSize(w, 256*1024) + h := sha256.New() + if _, err := bw.WriteString(header(title, serial)); err != nil { + return 0, "", err + } + write := func(owner string) error { + line := owner + "\tCNAME\t.\n" + h.Write([]byte(line)) + records++ + _, err := bw.WriteString(line) + return err + } + for _, n := range names { + f := l.Domains[n] + if f&Exact != 0 { + if err := write(n); err != nil { + return 0, "", err + } + } + if f&Below != 0 { + if err := write("*." + n); err != nil { + return 0, "", err + } + } + } + if err := bw.Flush(); err != nil { + return 0, "", err + } + return records, hex.EncodeToString(h.Sum(nil)), nil +} + +// MatchFeed reports which entry of a list's zone file blocks name: the +// name itself, or a wildcard of one of its ancestors (the closest). +// Lists are compacted, so no name exists below a wildcard and the +// closest wildcard ancestor is the one named uses. +func MatchFeed(path, name string) (owner string, ok bool, err error) { + want := map[string]int{name: 0} + depth := 1 + for a := name; ; depth++ { + i := strings.IndexByte(a, '.') + if i < 0 { + break + } + a = a[i+1:] + want["*."+a] = depth + } + f, err := os.Open(path) + if err != nil { + return "", false, err + } + defer f.Close() + best := -1 + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + o, _, found := strings.Cut(line, "\t") + if !found { + continue + } + if d, hit := want[o]; hit && (best < 0 || d < best) { + best, owner = d, o + } + } + return owner, best >= 0, sc.Err() +} diff --git a/internal/config/config.go b/internal/config/config.go index e7c4731..5b1867c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,18 +25,39 @@ type Config struct { TrustedProxies []string `yaml:"trusted_proxies"` // Socket is a local unix socket for the API, where root and jarld's // own user need no token ("" for none). - Socket string `yaml:"socket"` - Database string `yaml:"database"` - StagingDir string `yaml:"staging_dir"` - Log Log `yaml:"log"` - Session Session `yaml:"session"` - TOTP TOTP `yaml:"totp"` - BIND BIND `yaml:"bind"` - Audit Audit `yaml:"audit"` - Backup Backup `yaml:"backup"` - Stats Stats `yaml:"stats"` - ACME ACME `yaml:"acme"` - Porkbun Porkbun `yaml:"porkbun"` + Socket string `yaml:"socket"` + Database string `yaml:"database"` + StagingDir string `yaml:"staging_dir"` + Log Log `yaml:"log"` + Session Session `yaml:"session"` + TOTP TOTP `yaml:"totp"` + BIND BIND `yaml:"bind"` + Audit Audit `yaml:"audit"` + Backup Backup `yaml:"backup"` + Stats Stats `yaml:"stats"` + ACME ACME `yaml:"acme"` + Porkbun Porkbun `yaml:"porkbun"` + Blocking Blocking `yaml:"blocking"` +} + +// Blocking configures blocklist downloads (docs/blocking.md). The lists +// and rules themselves are configuration in the database. +type Blocking struct { + // Refresh is how often each list is checked for changes; a list is + // only downloaded again when it changed. + Refresh Duration `yaml:"refresh"` + MaxDownloadMB int `yaml:"max_download_mb"` + MaxEntries int `yaml:"max_entries"` +} + +func (b *Blocking) check() error { + if b.Refresh < Duration(time.Hour) { + return fmt.Errorf("refresh must be at least 1h (list hosts ask not to be polled more often)") + } + if b.MaxDownloadMB < 1 || b.MaxEntries < 1 { + return fmt.Errorf("max_download_mb and max_entries must be at least 1") + } + return nil } // Porkbun lets Jarl manage domains' name servers, glue records and DNSSEC @@ -300,8 +321,9 @@ func Default() *Config { Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)}, TopN: 100, }, - ACME: ACME{HTTPReq: HTTPReq{RequireHTTPS: true}}, - Backup: Backup{Dir: "/var/backups/jarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7, WarnAfter: Duration(2 * 24 * time.Hour)}, + ACME: ACME{HTTPReq: HTTPReq{RequireHTTPS: true}}, + Blocking: Blocking{Refresh: Duration(24 * time.Hour), MaxDownloadMB: 64, MaxEntries: 1_000_000}, + Backup: Backup{Dir: "/var/backups/jarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7, WarnAfter: Duration(2 * 24 * time.Hour)}, } } @@ -369,6 +391,9 @@ func (c *Config) check() error { if err := c.Porkbun.check(); err != nil { return fmt.Errorf("porkbun: %w", err) } + if err := c.Blocking.check(); err != nil { + return fmt.Errorf("blocking: %w", err) + } switch c.Log.Format { case "text", "json": default: @@ -600,4 +625,12 @@ porkbun: api_key: "" # pk1_... secret_api_key: "" # sk1_... domains: [] # only these domains (default: any Jarl zone the key reaches) + +# Downloading blocklists (docs/blocking.md). The lists and rules are set up +# in the web UI or with 'jarl blocklist' and 'jarl block'; this only sets +# how jarld downloads them. +blocking: + refresh: 24h # check each list for changes this often (at least 1h) + max_download_mb: 64 # refuse larger downloads + max_entries: 1000000 # refuse lists with more domains ` diff --git a/internal/engine/blocking_test.go b/internal/engine/blocking_test.go new file mode 100644 index 0000000..75907d7 --- /dev/null +++ b/internal/engine/blocking_test.go @@ -0,0 +1,231 @@ +package engine + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + + "jarl/internal/blocking" + "jarl/internal/model" +) + +func blockRule(name, action string, exact bool, views ...string) Request { + in := BlockRuleInput{Name: &name, Action: &action, Exact: &exact} + if len(views) > 0 { + in.Views = &views + } + return NewRequest(OpBlockRuleCreate, 0, in) +} + +func blocklist(name, url string, views ...string) Request { + in := BlocklistInput{Name: &name, URL: &url} + if len(views) > 0 { + in.Views = &views + } + return NewRequest(OpBlocklistCreate, 0, in) +} + +// listServer serves a blocklist with an ETag, and counts the downloads +// that sent content. +type listServer struct { + body atomic.Value + full atomic.Int32 + calls atomic.Int32 +} + +func newListServer(t *testing.T, body string) (*listServer, *httptest.Server) { + ls := &listServer{} + ls.body.Store(body) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ls.calls.Add(1) + b := ls.body.Load().(string) + etag := `"` + string(rune('a'+len(b)%26)) + "x" + string(rune('a'+len(b)/26%26)) + `"` + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", etag) + ls.full.Add(1) + w.Write([]byte(b)) + })) + t.Cleanup(srv.Close) + return ls, srv +} + +func TestBlockingRender(t *testing.T) { + env := newEnv(t) + setupZone(env) + env.submit(env.admin, + NewRequest(OpViewCreate, 0, ViewInput{Name: model.Ptr("internal"), MatchClients: &[]string{"10.0.0.0/8"}, AdoptZones: true}), + NewRequest(OpViewCreate, 0, ViewInput{Name: model.Ptr("external")}), + ) + st := env.state() + ext := st.ViewByName("external") + env.submit(env.admin, NewRequest(OpOptionCreate, 0, OptionInput{Scope: model.Ptr(model.ScopeView), ScopeID: &ext.ID, Name: model.Ptr("recursion"), Value: model.Ptr("no")})) + + _, srv := newListServer(t, "||ads.example^\n") + res := env.submit(env.admin, + blocklist("ads", srv.URL+"/list.txt"), + blockRule("tracker.example", "block", false), + blockRule("cdn.tracker.example", "allow", true), + ) + if res.Ops[1].Summary != "block tracker.example (and below)" || res.Ops[2].Summary != "allow cdn.tracker.example" { + t.Errorf("summaries: %q, %q", res.Ops[1].Summary, res.Ops[2].Summary) + } + conf, _ := os.ReadFile(filepath.Join(env.dir, "etc/named.conf")) + c := string(conf) + // Automatic views: only internal offers recursion. + internal := c[strings.Index(c, `view "internal"`):strings.Index(c, `view "external"`)] + for _, want := range []string{ + "response-policy {\n\t\tzone \"rules.jarl.invalid\" min-update-interval 0 log no;\n\t\tzone \"ads.blocklist.jarl.invalid\" min-update-interval 0 log no;\n\t} qname-wait-recurse no break-dnssec yes;", + `zone "rules.jarl.invalid" {`, + "allow-query { none; };", + } { + if !strings.Contains(internal, want) { + t.Errorf("view internal lacks %q:\n%s", want, internal) + } + } + if strings.Contains(c[strings.Index(c, `view "external"`):], "response-policy") { + t.Error("view external, without recursion, got blocking") + } + rules, err := os.ReadFile(filepath.Join(env.dir, "zones/internal/rules.jarl.invalid.rpz")) + if err != nil || !strings.Contains(string(rules), "cdn.tracker.example\tCNAME\trpz-passthru.\n*.cdn.tracker.example\tCNAME\t.\n") { + t.Errorf("rules zone: %v\n%s", err, rules) + } + // The list's file exists as an empty policy zone until downloaded. + b := env.state().BlocklistByName("ads") + listFile := filepath.Join(env.dir, "cache/blocklists/list-"+strconv.FormatInt(b.ID, 10)+".rpz") + if f, err := os.ReadFile(listFile); err != nil || !strings.Contains(string(f), "not downloaded yet") { + t.Errorf("placeholder: %v %s", err, f) + } + + // A view a list or rule names can't be deleted. + res = env.submit(env.admin, NewRequest(OpBlocklistUpdate, b.ID, BlocklistInput{Views: &[]string{"external"}})) + if len(res.Problems) != 1 || res.Problems[0].Code != "blocking_no_recursion" { + t.Errorf("a list in a view without recursion: %+v", res.Problems) + } + if e := env.submitErr(env.admin, NewRequest(OpViewDelete, ext.ID, DeleteInput{Cascade: true})); e.Kind != KindConflict || !strings.Contains(e.Message, "blocklist ads") { + t.Errorf("deleting a used view: %+v", e) + } + st = env.state() + // A second rule for the same name and view is refused. + if e := env.submitErr(env.admin, blockRule("tracker.example", "allow", true)); e.Kind != KindValidation || !strings.Contains(e.Message, "two rules for tracker.example") { + t.Errorf("duplicate rule: %+v", e) + } + // So is a response-policy option where Jarl writes one. + in := st.ViewByName("internal") + if e := env.submitErr(env.admin, NewRequest(OpOptionCreate, 0, OptionInput{Scope: model.Ptr(model.ScopeView), ScopeID: &in.ID, Name: model.Ptr("response-policy"), Value: model.Ptr(`{ zone "x"; }`)})); e.Kind != KindValidation { + t.Errorf("response-policy option: %+v", e) + } + // Bad input. + for _, req := range []Request{blocklist("Bad Name", srv.URL), blocklist("ok", "ftp://x/list"), blockRule("com", "block", false), blockRule("a.example", "deny", false), blockRule("a.example", "block", false, "nope")} { + if e := env.submitErr(env.admin, req); e.Kind != KindBadRequest { + t.Errorf("%s: %+v", req.Data, e) + } + } +} + +func TestBlocklistRefresh(t *testing.T) { + env := newEnv(t) + setupZone(env) + ls, srv := newListServer(t, "# test\n||ads.example^\n0.0.0.0 tracker.example\nnot a domain line\n") + env.submit(env.admin, blocklist("ads", srv.URL), blockRule("good.ads.example", "allow", false)) + b := env.state().BlocklistByName("ads") + ctx := context.Background() + + env.e.RefreshDue(ctx) + s, _ := env.st.BlocklistStatuses(ctx) + st := s[b.ID] + if st == nil || st.Error != "" || st.Entries != 2 || st.Records != 3 || st.Skipped != 1 || st.UpdatedAt == nil { + t.Fatalf("status after the first download: %+v", st) + } + file := env.e.cfg.Paths.BlocklistFile(b.ID) + if f, _ := os.ReadFile(file); !strings.Contains(string(f), "*.ads.example\tCNAME\t.\n") || !strings.Contains(string(f), "tracker.example\tCNAME\t.\n") { + t.Errorf("list zone:\n%s", f) + } + // Not due again: nothing is fetched. + env.e.RefreshDue(ctx) + if ls.calls.Load() != 1 { + t.Errorf("downloads after a second RefreshDue: %d", ls.calls.Load()) + } + // A cleared cache leaves an empty placeholder: downloaded again at once. + os.Remove(file) + env.e.RefreshDue(ctx) + if ls.calls.Load() != 2 { + t.Errorf("downloads after the file was lost: %d", ls.calls.Load()) + } + if f, _ := os.ReadFile(file); !strings.Contains(string(f), "*.ads.example\tCNAME\t.\n") { + t.Errorf("list zone after the file was lost:\n%s", f) + } + // Unchanged on the server: 304, no rewrite. + firstHash := st.SHA256 + fullBefore := ls.full.Load() + env.e.RefreshBlocklist(ctx, b.ID, false) + if ls.full.Load() != fullBefore { + t.Errorf("a conditional download fetched the list again") + } + // Changed: rewritten. + ls.body.Store("||ads.example^\n||more.example^\n") + st, err := env.e.RefreshBlocklist(ctx, b.ID, false) + if err != nil || st.Entries != 2 || st.Records != 4 || st.SHA256 == firstHash { + t.Errorf("after a change: %+v, %v", st, err) + } + // A broken download keeps the previous version and reports why. + before, _ := os.ReadFile(file) + ls.body.Store("Not found\n") + st, err = env.e.RefreshBlocklist(ctx, b.ID, true) + if err == nil || !strings.Contains(st.Error, "no domains found") { + t.Errorf("garbage: %+v, %v", st, err) + } + if after, _ := os.ReadFile(file); string(after) != string(before) { + t.Error("a failed update changed the list's file") + } + // A failed attempt is retried after RetryAfter, not at every check. + calls := ls.calls.Load() + env.e.RefreshDue(ctx) + if ls.calls.Load() != calls { + t.Error("a failed list was tried again before RetryAfter") + } + // Too big. + env.e.Limits.MaxBytes = 10 + ls.body.Store("||ads.example^\n||more.example^\n||third.example^\n") + if st, _ = env.e.RefreshBlocklist(ctx, b.ID, true); !strings.Contains(st.Error, "larger than") { + t.Errorf("size limit: %+v", st) + } + env.e.Limits = DefaultBlocklistLimits() + env.e.RefreshBlocklist(ctx, b.ID, true) + + // What blocks what. + check := func(name string) blocking.Match { + m, err := env.e.CheckBlocking(env.state(), name) + if err != nil || len(m) != 1 { + t.Fatalf("check %s: %+v %v", name, m, err) + } + return m[0] + } + if m := check("x.ads.example"); m.Result != "blocked" || m.List != "ads" || m.Entry != "*.ads.example" { + t.Errorf("x.ads.example: %+v", m) + } + if m := check("a.good.ads.example"); m.Result != "allowed" || m.RuleID == 0 || m.Entry != "*.good.ads.example" { + t.Errorf("a.good.ads.example: %+v", m) + } + if m := check("fine.example"); m.Result != "none" { + t.Errorf("fine.example: %+v", m) + } + + // Deleting the list removes its file and status. + env.submit(env.admin, NewRequest(OpBlocklistDelete, b.ID, nil)) + env.e.RefreshDue(ctx) + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Errorf("the deleted list's file is still there: %v", err) + } + if s, _ := env.st.BlocklistStatuses(ctx); s[b.ID] != nil { + t.Error("the deleted list's status is still there") + } +} diff --git a/internal/engine/blocklists.go b/internal/engine/blocklists.go new file mode 100644 index 0000000..49684bd --- /dev/null +++ b/internal/engine/blocklists.go @@ -0,0 +1,403 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "jarl/internal/blocking" + "jarl/internal/model" + "jarl/internal/render" + "jarl/internal/version" +) + +// BlocklistLimits bound blocklist downloads (blocking in jarl.yaml). +type BlocklistLimits struct { + Refresh time.Duration // how often a list is checked for changes + RetryAfter time.Duration // after a failed attempt + MaxBytes int64 // per download + MaxEntries int // domains per list + Timeout time.Duration // per download +} + +// DefaultBlocklistLimits are the limits without configuration. +func DefaultBlocklistLimits() BlocklistLimits { + return BlocklistLimits{Refresh: 24 * time.Hour, RetryAfter: time.Hour, MaxBytes: 64 << 20, MaxEntries: 1_000_000, Timeout: 2 * time.Minute} +} + +// KickBlocklists asks the download loop to look for work now (a list was +// added or changed). +func (e *Engine) KickBlocklists() { + select { + case e.blKick <- struct{}{}: + default: + } +} + +// BlocklistKicks delivers KickBlocklists' signals to the download loop. +func (e *Engine) BlocklistKicks() <-chan struct{} { return e.blKick } + +// ensureBlocklistFiles creates missing blocklist zone files as empty +// policy zones, so named can load its configuration before a list has +// been downloaded. Existing files are never touched. +func (e *Engine) ensureBlocklistFiles(paths []string) error { + var errs []error + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + continue + } + if err := e.deploy.writeNew(path, []byte(blocking.Placeholder()), 0o640); err != nil && !errors.Is(err, fs.ErrExist) { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// RefreshDue downloads the blocklists that are due: never tried, URL +// changed, file missing or empty (the cache was cleared), last check older +// than Refresh, or a failed attempt older than RetryAfter. Lists that are +// disabled or apply to no view are skipped. It also removes the files and +// status of deleted lists. +func (e *Engine) RefreshDue(ctx context.Context) { + st, err := e.store.LoadState(ctx) + if err != nil { + e.log.Warn("blocklists: loading the configuration failed", "err", err) + return + } + statuses, err := e.store.BlocklistStatuses(ctx) + if err != nil { + e.log.Warn("blocklists: loading their status failed", "err", err) + return + } + out := render.Render(st, e.cfg.Paths, e.cfg.Extras) + if err := e.ensureBlocklistFiles(out.Blocklists); err != nil { + e.log.Warn("blocklists: creating empty zone files failed", "err", err) + } + e.cleanBlocklists(ctx, st, statuses) + now := e.Now() + for _, b := range st.SortedBlocklists() { + if b.Disabled || len(st.BlockingViews(b.ViewIDs)) == 0 { + continue + } + s := statuses[b.ID] + due := s == nil || s.URL != b.URL || s.CheckedAt == nil + if !due { + wait := e.Limits.Refresh + if s.Error != "" { + wait = e.Limits.RetryAfter + } + due = now.Sub(*s.CheckedAt) >= wait || s.Error == "" && isPlaceholder(e.cfg.Paths.BlocklistFile(b.ID)) + } + if !due { + continue + } + if ctx.Err() != nil { + return + } + if _, err := e.RefreshBlocklist(ctx, b.ID, false); err != nil && ctx.Err() == nil { + e.log.Warn("blocklist update failed; named keeps using the previous version", "list", b.Name, "err", err) + } + } +} + +// isPlaceholder reports whether a list's zone file is missing or still the +// empty policy zone written before its download. +func isPlaceholder(path string) bool { + f, err := os.Open(path) + if err != nil { + return true + } + defer f.Close() + head := make([]byte, len(blocking.Placeholder())) + n, _ := io.ReadFull(f, head) + return string(head[:n]) == blocking.Placeholder() +} + +// cleanBlocklists removes the zone files and status of deleted lists. +func (e *Engine) cleanBlocklists(ctx context.Context, st *model.State, statuses map[int64]*model.BlocklistStatus) { + for id := range statuses { + if st.Blocklists[id] == nil { + e.store.DeleteBlocklistStatus(ctx, id) + } + } + dir := e.cfg.Paths.BlocklistsDir() + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, ent := range entries { + n := ent.Name() + idText, ok := strings.CutPrefix(strings.TrimSuffix(n, ".rpz"), "list-") + id, err := strconv.ParseInt(idText, 10, 64) + if !ok || !strings.HasSuffix(n, ".rpz") || err != nil || st.Blocklists[id] != nil { + continue + } + if err := os.Remove(filepath.Join(dir, n)); err == nil { + e.log.Info("removed the zone file of a deleted blocklist", "file", n) + } + } +} + +// errNotModified means the server said the list hasn't changed. +var errNotModified = errors.New("not modified") + +// RefreshBlocklist downloads a list now and, when its content changed, +// replaces its zone file and makes named reload it. force downloads even +// when the server says nothing changed. A failure keeps the previous +// version. It returns the new status. +func (e *Engine) RefreshBlocklist(ctx context.Context, id int64, force bool) (*model.BlocklistStatus, error) { + e.blMu.Lock() + defer e.blMu.Unlock() + st, err := e.store.LoadState(ctx) + if err != nil { + return nil, err + } + b := st.Blocklists[id] + if b == nil { + return nil, errf(KindNotFound, "blocklist %d does not exist", id) + } + statuses, err := e.store.BlocklistStatuses(ctx) + if err != nil { + return nil, err + } + prev := statuses[id] + if prev == nil { + prev = &model.BlocklistStatus{ID: id} + } + file := e.cfg.Paths.BlocklistFile(id) + now := e.Now() + next := *prev + next.CheckedAt = &now + conditional := !force && prev.URL == b.URL && prev.SHA256 != "" && !isPlaceholder(file) + err = e.downloadBlocklist(ctx, st, b, &next, conditional, file) + next.URL, next.Error = b.URL, "" + if err != nil && !errors.Is(err, errNotModified) { + next.Error = err.Error() + } + if serr := e.store.SetBlocklistStatus(ctx, &next); serr != nil && err == nil { + err = serr + } + if errors.Is(err, errNotModified) { + err = nil + } + return &next, err +} + +// downloadBlocklist fetches, parses, checks and installs a list, filling +// in s. It returns errNotModified when there is nothing to do. +func (e *Engine) downloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist, s *model.BlocklistStatus, conditional bool, file string) error { + ctx, cancel := context.WithTimeout(ctx, e.Limits.Timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.URL, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Jarl/"+version.Short()+" (blocklist download)") + if conditional { + if s.ETag != "" { + req.Header.Set("If-None-Match", s.ETag) + } + if s.LastModified != "" { + req.Header.Set("If-Modified-Since", s.LastModified) + } + } + client := e.HTTP + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotModified && conditional { + return errNotModified + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download failed: the server answered %s", resp.Status) + } + zone := blocking.ListZone(b.Name) + body := &limitedReader{r: resp.Body, left: e.Limits.MaxBytes} + l, err := blocking.Parse(body, blocking.Limits{MaxEntries: e.Limits.MaxEntries, MaxNameLen: 253 - len(zone) - 3}) + if body.over { + return fmt.Errorf("the list is larger than %d MB (blocking.max_download_mb in jarl.yaml)", e.Limits.MaxBytes>>20) + } + if errors.Is(err, blocking.ErrTooManyEntries) { + return fmt.Errorf("%v (blocking.max_entries in jarl.yaml)", err) + } + if err != nil { + return fmt.Errorf("download failed: %w", err) + } + if len(l.Domains) == 0 { + return fmt.Errorf("no domains found in %d lines: is this a blocklist in hosts, domains or Adblock format?", l.Lines) + } + entries, skipped := len(l.Domains), l.Skipped + blocking.Compact(l) + + dir := filepath.Dir(file) + if err := e.deploy.EnsureDir(dir, e.deploy.dirMode(dir)); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".list-*.rpz") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) // after a successful rename, there's nothing to remove + records, hash, err := blocking.FeedZone(tmp, fmt.Sprintf("Blocklist %s from %s, %s", b.Name, b.URL, e.Now().UTC().Format(time.RFC3339)), uint32(e.Now().Unix()), l) + if cerr := tmp.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + s.ETag, s.LastModified = resp.Header.Get("ETag"), resp.Header.Get("Last-Modified") + s.Entries, s.Skipped = entries, skipped + if hash == s.SHA256 && s.URL == b.URL && !isPlaceholder(file) { + return errNotModified // same content: named keeps what it has + } + if e.tools != nil && !e.cfg.SkipCheck { + res, err := e.tools.CheckZone(ctx, zone, tmpName) + if err != nil { + return fmt.Errorf("running named-checkzone failed: %w", err) + } + if !res.OK { + msg := "named-checkzone rejected the list's zone" + for _, m := range res.Messages { + if m.Error { + msg += ": " + m.Text + break + } + } + return errors.New(msg) + } + } + if err := e.deploy.finish(tmpName, file, 0o640); err != nil { + return err + } + now := e.Now() + s.UpdatedAt, s.Records, s.SHA256 = &now, records, hash + e.log.Info("blocklist updated", "list", b.Name, "domains", entries, "records", records, "skipped_lines", skipped) + return e.reloadBlocklist(ctx, st, b) +} + +// reloadBlocklist makes named load a list's new file in every view that +// uses it. +func (e *Engine) reloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist) error { + if e.cfg.NoReload || e.tools == nil { + return nil + } + if adopted, err := e.Adopted(); err != nil || !adopted { + return nil // the file is ready for when named loads Jarl's configuration + } + for _, z := range render.PolicyZones(st, e.cfg.Paths) { + if z.ListID != b.ID { + continue + } + if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil { + return fmt.Errorf("the list was downloaded, but named didn't reload it: %w", err) + } + } + return nil +} + +// limitedReader reads at most left bytes and notes when there was more. +type limitedReader struct { + r io.Reader + left int64 + over bool +} + +func (l *limitedReader) Read(p []byte) (int, error) { + if l.left <= 0 { + var one [1]byte + if n, _ := l.r.Read(one[:]); n > 0 { + l.over = true + } + return 0, io.EOF + } + if int64(len(p)) > l.left { + p = p[:l.left] + } + n, err := l.r.Read(p) + l.left -= int64(n) + return n, err +} + +// CheckBlocking reports, for every view with blocking, whether name is +// blocked or allowed there, and by which rule or list. It looks names up +// the way named does: rules first, then the lists in order. +func (e *Engine) CheckBlocking(st *model.State, name string) ([]blocking.Match, error) { + n, err := blocking.NormalizeName(name) + if err != nil { + return nil, errf(KindBadRequest, "%v", err) + } + var out []blocking.Match + feeds := map[string]string{} // file -> matching entry ("" none) + var cur *blocking.Match + for _, z := range render.PolicyZones(st, e.cfg.Paths) { + if cur == nil || cur.View != z.View { + out = append(out, blocking.Match{View: z.View, Result: "none"}) + cur = &out[len(out)-1] + } + if cur.Result != "none" { + continue // an earlier zone decided + } + if z.ListID == 0 { + owner, act, ok := blocking.MatchRules(blocking.RuleEntries(render.ViewRules(st, z.ViewID)), n) + if ok { + cur.Entry, cur.RuleID = owner, ruleFor(st, z.ViewID, owner) + cur.Result = map[string]string{blocking.ActionBlock: "blocked", blocking.ActionAllow: "allowed"}[act] + } + continue + } + entry, seen := feeds[z.File] + if !seen { + owner, ok, err := blocking.MatchFeed(z.File, n) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if ok { + entry = owner + } + feeds[z.File] = entry + } + if entry != "" { + cur.Result, cur.Entry, cur.List = "blocked", entry, st.Blocklists[z.ListID].Name + } + } + return out, nil +} + +// ruleFor finds the rule behind a rules-zone entry: the rule for the +// entry's name, or (for entries added so the most specific rule wins) the +// closest rule above it that covers the names below it. +func ruleFor(st *model.State, viewID int64, owner string) int64 { + name := strings.TrimPrefix(owner, "*.") + wild := name != owner + var best *model.BlockRule + for _, r := range st.SortedBlockRules() { + if !slices.Contains(st.BlockingViews(r.ViewIDs), viewID) { + continue + } + if r.Name == name && (!wild || !r.Exact) { + return r.ID + } + if !r.Exact && strings.HasSuffix(name, "."+r.Name) && (best == nil || len(r.Name) > len(best.Name)) { + best = r + } + } + if best != nil { + return best.ID + } + return 0 +} diff --git a/internal/engine/blockops.go b/internal/engine/blockops.go new file mode 100644 index 0000000..da64e98 --- /dev/null +++ b/internal/engine/blockops.go @@ -0,0 +1,272 @@ +package engine + +import ( + "encoding/json" + "fmt" + "net/url" + "regexp" + "slices" + "strings" + + "jarl/internal/auth" + "jarl/internal/blocking" + "jarl/internal/model" +) + +// ---- blocking: blocklists and rules (docs/blocking.md) + +var blocklistNameRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,38}[a-z0-9])?$`) + +func blockingAction() auth.Action { return auth.Action{Resource: auth.ResBlocking, Write: true} } + +// viewIDs resolves the views of a blocklist or rule input: names or IDs, +// in view order, without duplicates. nil, nil means "not given". +func viewIDs(st *model.State, names *[]string, ids *[]int64) ([]int64, error) { + if names == nil && ids == nil { + return nil, nil + } + want := map[int64]bool{} + if names != nil { + for _, n := range *names { + v := st.ViewByName(strings.TrimSpace(n)) + if v == nil { + return nil, errf(KindBadRequest, "view %q does not exist", n) + } + want[v.ID] = true + } + } + if ids != nil { + for _, id := range *ids { + if st.Views[id] == nil { + return nil, errf(KindBadRequest, "view %d does not exist", id) + } + want[id] = true + } + } + out := []int64{} + for _, v := range st.OrderedViews() { + if want[v.ID] { + out = append(out, v.ID) + } + } + return out, nil +} + +// CheckBlocklistURL checks a blocklist URL: http or https, with a host. +func CheckBlocklistURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", fmt.Errorf("%q is not an http or https URL", raw) + } + return raw, nil +} + +func applyBlocklistPut(st *model.State, op model.Op, c *applyCtx) error { + if err := c.require(blockingAction()); err != nil { + return err + } + var in BlocklistInput + if err := decode(op.Data, &in); err != nil { + return err + } + var b *model.Blocklist + if op.Kind == OpBlocklistCreate { + if op.ID == 0 || st.Blocklists[op.ID] != nil { + return errf(KindConflict, "blocklist ID %d is already in use", op.ID) + } + if in.Name == nil || in.URL == nil { + return errf(KindBadRequest, "a blocklist needs a name and a URL") + } + b = &model.Blocklist{ID: op.ID, ViewIDs: []int64{}} + } else { + b = st.Blocklists[op.ID] + if b == nil { + return errf(KindNotFound, "blocklist %d does not exist", op.ID) + } + if err := checkVersion(op, b.Version, "blocklist "+b.Name); err != nil { + return err + } + } + if in.Name != nil { + name := strings.ToLower(strings.TrimSpace(*in.Name)) + if !blocklistNameRe.MatchString(name) { + return errf(KindBadRequest, "blocklist name %q: use 1 to 40 lowercase letters, digits and hyphens, like hagezi-pro", *in.Name) + } + if other := st.BlocklistByName(name); other != nil && other.ID != b.ID { + return errf(KindConflict, "blocklist %s already exists", name) + } + b.Name = name + } + if in.URL != nil { + u, err := CheckBlocklistURL(*in.URL) + if err != nil { + return errf(KindBadRequest, "%v", err) + } + b.URL = u + } + ids, err := viewIDs(st, in.Views, in.ViewIDs) + if err != nil { + return err + } + if ids != nil { + b.ViewIDs = ids + } + if in.Disabled != nil { + b.Disabled = *in.Disabled + } + if in.Comment != nil { + b.Comment = *in.Comment + } + b.Touch(c.user(), c.now) + st.Blocklists[b.ID] = b + return nil +} + +func applyBlocklistDelete(st *model.State, op model.Op, c *applyCtx) error { + if err := c.require(blockingAction()); err != nil { + return err + } + b := st.Blocklists[op.ID] + if b == nil { + return errf(KindNotFound, "blocklist %d does not exist", op.ID) + } + if err := checkVersion(op, b.Version, "blocklist "+b.Name); err != nil { + return err + } + delete(st.Blocklists, b.ID) + return nil +} + +func applyBlockRulePut(st *model.State, op model.Op, c *applyCtx) error { + if err := c.require(blockingAction()); err != nil { + return err + } + var in BlockRuleInput + if err := decode(op.Data, &in); err != nil { + return err + } + var r *model.BlockRule + if op.Kind == OpBlockRuleCreate { + if op.ID == 0 || st.BlockRules[op.ID] != nil { + return errf(KindConflict, "rule ID %d is already in use", op.ID) + } + if in.Name == nil { + return errf(KindBadRequest, "a rule needs a domain name") + } + r = &model.BlockRule{ID: op.ID, Action: model.BlockActionBlock, ViewIDs: []int64{}} + } else { + r = st.BlockRules[op.ID] + if r == nil { + return errf(KindNotFound, "rule %d does not exist", op.ID) + } + if err := checkVersion(op, r.Version, "rule "+r.Name); err != nil { + return err + } + } + if in.Name != nil { + n, err := blocking.NormalizeName(*in.Name) + if err != nil { + return errf(KindBadRequest, "%v", err) + } + r.Name = n + } + if in.Action != nil { + a := strings.ToLower(strings.TrimSpace(*in.Action)) + if a != model.BlockActionBlock && a != model.BlockActionAllow { + return errf(KindBadRequest, "a rule's action is block or allow, not %q", *in.Action) + } + r.Action = a + } + if in.Exact != nil { + r.Exact = *in.Exact + } + ids, err := viewIDs(st, in.Views, in.ViewIDs) + if err != nil { + return err + } + if ids != nil { + r.ViewIDs = ids + } + if in.Comment != nil { + r.Comment = *in.Comment + } + r.Touch(c.user(), c.now) + st.BlockRules[r.ID] = r + return nil +} + +func applyBlockRuleDelete(st *model.State, op model.Op, c *applyCtx) error { + if err := c.require(blockingAction()); err != nil { + return err + } + r := st.BlockRules[op.ID] + if r == nil { + return errf(KindNotFound, "rule %d does not exist", op.ID) + } + if err := checkVersion(op, r.Version, "rule "+r.Name); err != nil { + return err + } + delete(st.BlockRules, r.ID) + return nil +} + +// blockingUsers lists the blocklists and rules that name a view. +func blockingUsers(st *model.State, viewID int64) []string { + var out []string + for _, b := range st.SortedBlocklists() { + if slices.Contains(b.ViewIDs, viewID) { + out = append(out, "blocklist "+b.Name) + } + } + for _, r := range st.SortedBlockRules() { + if slices.Contains(r.ViewIDs, viewID) { + out = append(out, ruleLabel(r)) + } + } + return out +} + +// ruleLabel describes a rule: "block ads.example (and below)". +func ruleLabel(r *model.BlockRule) string { + s := r.Action + " " + r.Name + if !r.Exact { + s += " (and below)" + } + return s +} + +// ---- expansion + +func (x *expander) blocklistCreate(req Request) ([]model.Op, error) { + return x.simpleCreate(req, OpBlocklistCreate, func(data json.RawMessage) (string, error) { + var in BlocklistInput + if err := decode(data, &in); err != nil { + return "", err + } + if in.Name == nil || in.URL == nil { + return "", errf(KindBadRequest, "a blocklist needs a name and a URL") + } + return "add blocklist " + *in.Name + " (" + *in.URL + ")", nil + }) +} + +func (x *expander) blockRuleCreate(req Request) ([]model.Op, error) { + return x.simpleCreate(req, OpBlockRuleCreate, func(data json.RawMessage) (string, error) { + var in BlockRuleInput + if err := decode(data, &in); err != nil { + return "", err + } + if in.Name == nil { + return "", errf(KindBadRequest, "a rule needs a domain name") + } + r := &model.BlockRule{Name: strings.ToLower(strings.TrimSuffix(strings.TrimSpace(*in.Name), ".")), Action: model.BlockActionBlock} + if in.Action != nil { + r.Action = strings.ToLower(*in.Action) + } + if in.Exact != nil { + r.Exact = *in.Exact + } + return ruleLabel(r), nil + }) +} diff --git a/internal/engine/deploy.go b/internal/engine/deploy.go index 5fa7755..bf8f370 100644 --- a/internal/engine/deploy.go +++ b/internal/engine/deploy.go @@ -183,6 +183,48 @@ func (d *Deployer) writeAtomic(path string, content []byte, mode fs.FileMode) er return nil } +// writeNew creates a file that doesn't exist yet (fs.ErrExist if it +// does), with the deploy group. +func (d *Deployer) writeNew(path string, content []byte, mode fs.FileMode) error { + dir := filepath.Dir(path) + if err := d.EnsureDir(dir, d.dirMode(dir)); err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + _, err = f.Write(content) + if gid, gerr := d.groupID(); err == nil && gerr == nil && gid >= 0 { + err = f.Chown(-1, gid) + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(path) + } + return err +} + +// finish gives a temporary file written next to path its mode and the +// deploy group, and moves it into place. +func (d *Deployer) finish(tmp, path string, mode fs.FileMode) error { + if err := os.Chmod(tmp, mode); err != nil { + return err + } + gid, err := d.groupID() + if err != nil { + return err + } + if gid >= 0 { + if err := os.Chown(tmp, -1, gid); err != nil { + return fmt.Errorf("chgrp %s: %w", path, err) + } + } + return os.Rename(tmp, path) +} + // Install writes files and removes paths, returning a backup that undoes // the changes. On error the files already written are restored. func (d *Deployer) Install(files []*render.File, remove []string, dirs []string) (*Backup, error) { diff --git a/internal/engine/diffops.go b/internal/engine/diffops.go index e10189f..d2972c1 100644 --- a/internal/engine/diffops.go +++ b/internal/engine/diffops.go @@ -101,7 +101,35 @@ func DiffOps(from, to *model.State) []model.Op { add(OpRecordUpdate, k, o.Version, in, fmt.Sprintf("restore %s %s %s", n.Name, n.Type, n.Data)) } } + for _, k := range sortedKeys(to.Blocklists) { + n := to.Blocklists[k] + in := BlocklistInput{Name: &n.Name, URL: &n.URL, ViewIDs: model.Ptr(append([]int64{}, n.ViewIDs...)), Disabled: &n.Disabled, Comment: &n.Comment} + if o, ok := from.Blocklists[k]; !ok { + add(OpBlocklistCreate, k, 0, in, "add blocklist "+n.Name) + } else if !sameObject(o, n) { + add(OpBlocklistUpdate, k, o.Version, in, "restore blocklist "+n.Name) + } + } + for _, k := range sortedKeys(to.BlockRules) { + n := to.BlockRules[k] + in := BlockRuleInput{Name: &n.Name, Action: &n.Action, Exact: &n.Exact, ViewIDs: model.Ptr(append([]int64{}, n.ViewIDs...)), Comment: &n.Comment} + if o, ok := from.BlockRules[k]; !ok { + add(OpBlockRuleCreate, k, 0, in, "add rule: "+ruleLabel(n)) + } else if !sameObject(o, n) { + add(OpBlockRuleUpdate, k, o.Version, in, "restore rule: "+ruleLabel(n)) + } + } // Deletions, most dependent objects first. + for _, k := range sortedKeys(from.BlockRules) { + if _, ok := to.BlockRules[k]; !ok { + add(OpBlockRuleDelete, k, from.BlockRules[k].Version, nil, "delete rule: "+ruleLabel(from.BlockRules[k])) + } + } + for _, k := range sortedKeys(from.Blocklists) { + if _, ok := to.Blocklists[k]; !ok { + add(OpBlocklistDelete, k, from.Blocklists[k].Version, nil, "delete blocklist "+from.Blocklists[k].Name) + } + } for _, k := range sortedKeys(from.Records) { if _, ok := to.Records[k]; !ok { o := from.Records[k] diff --git a/internal/engine/engine.go b/internal/engine/engine.go index c4c6b1a..df1c273 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -14,8 +14,10 @@ import ( "fmt" "io/fs" "log/slog" + "net/http" "os" "path/filepath" + "slices" "sort" "strings" "sync" @@ -23,6 +25,7 @@ import ( "jarl/internal/auth" "jarl/internal/bind" + "jarl/internal/blocking" "jarl/internal/dnsname" "jarl/internal/model" "jarl/internal/namedconf" @@ -64,6 +67,12 @@ type Engine struct { deploy *Deployer mu sync.Mutex // serializes applies Now func() time.Time + + // Blocklist downloads (blocklists.go). + blMu sync.Mutex // serializes downloads + blKick chan struct{} // a list was added or changed + HTTP *http.Client // for downloads; nil: a default client + Limits BlocklistLimits } // New creates an engine. @@ -78,6 +87,8 @@ func New(st *store.Store, tools *bind.Tools, cfg Config, log *slog.Logger) *Engi store: st, tools: tools, cfg: cfg, log: log, deploy: &Deployer{Group: cfg.FileGroup, NamedWritable: []string{cfg.Paths.ZoneDir, cfg.Paths.CacheDir}}, Now: time.Now, + blKick: make(chan struct{}, 1), + Limits: DefaultBlocklistLimits(), } } @@ -383,6 +394,7 @@ type plan struct { drift []DriftEntry configChanged bool reloadZones []*model.Zone + reloadPolicy []render.PolicyZone // rules zones whose file changed verify map[int64]uint32 newZones bool backup *Backup @@ -446,7 +458,7 @@ func (e *Engine) makePlan(ctx context.Context, base, work *model.State, user str } } for path, df := range deployed { - if pl.out.File(path) != nil || df.Kind != render.KindZone { + if pl.out.File(path) != nil || df.Kind != render.KindZone && df.Kind != render.KindPolicy { continue } old, existed, _ := readFile(path) @@ -484,6 +496,16 @@ func (e *Engine) makePlan(ctx context.Context, base, work *model.State, user str if len(work.Zones) != len(base.Zones) { pl.configChanged = true } + // Rules zones whose content changed. New ones come with a change to + // named.conf, and reconfig loads them. + for _, z := range pl.out.Policy { + if z.ListID != 0 || !changedPaths[z.File] { + continue + } + if slices.Contains(baseOut.Policy, z) { + pl.reloadPolicy = append(pl.reloadPolicy, z) + } + } if !e.cfg.SkipCheck { if err := e.stagingCheck(ctx, pl); err != nil { @@ -519,6 +541,16 @@ func (e *Engine) stagingCheck(ctx context.Context, pl *plan) error { return err } } + // Blocklists as empty policy zones: their content is checked when + // they are downloaded. + for _, path := range sout.Blocklists { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + if err := os.WriteFile(path, []byte(blocking.Placeholder()), 0o640); err != nil { + return err + } + } main, err := e.stagingMain(dir, sp.MainConfig()) if err != nil { return err @@ -796,6 +828,12 @@ func (e *Engine) applyOps(ctx context.Context, cs *model.Changeset, reqs []Reque res.ChangesetID = cs.ID res.Applied = true e.removeDerived(pl) + for _, op := range cs.Ops { + if strings.HasPrefix(op.Kind, "blocklist.") { + e.KickBlocklists() + break + } + } e.log.Info("changeset applied", "changeset", cs.ID, "user", p.Name(), "ops", len(cs.Ops), "files", len(res.Files), "description", cs.Description) return res, nil } @@ -829,6 +867,9 @@ func (e *Engine) deployAndReload(ctx context.Context, pl *plan, res *Result, rel return &Error{Kind: KindBIND, Message: "writing configuration files failed (nothing was changed): " + err.Error()} } pl.backup = backup + if err := e.ensureBlocklistFiles(pl.out.Blocklists); err != nil { + e.log.Warn("creating a blocklist's empty zone file failed", "err", err) + } pl.reloaded = reload if !reload { return nil @@ -860,6 +901,11 @@ func (e *Engine) rollback(ctx context.Context, pl *plan) { e.log.Error("rndc reload after restore failed", "zone", z.Name, "err", err) } } + for _, z := range pl.reloadPolicy { + if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil { + e.log.Error("rndc reload after restore failed", "zone", z.Name, "err", err) + } + } } // namedView returns the view name to pass to rndc for a zone ("" without @@ -885,6 +931,16 @@ func (e *Engine) reload(ctx context.Context, pl *plan) ([]string, error) { } done = append(done, "reload "+zoneDesc(pl.work, z)) } + for _, z := range pl.reloadPolicy { + label := "blocking rules" + if z.View != "" { + label += " [view " + z.View + "]" + } + if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil { + return done, fmt.Errorf("reloading %s failed: %v", label, err) + } + done = append(done, "reload "+label) + } // named reloads zones asynchronously and reports load errors only in // its log, so wait until it serves the new serials. deadline := time.Now().Add(e.cfg.ReloadTimeout) @@ -1003,6 +1059,12 @@ func (e *Engine) Resync(ctx context.Context, p *auth.Principal, opts ApplyOption pl.verify[z.ID] = z.SOA.Serial } } + pl.reloadPolicy = nil + for _, z := range pl.out.Policy { + if z.ListID == 0 { + pl.reloadPolicy = append(pl.reloadPolicy, z) + } + } reload := !e.cfg.NoReload && !opts.NoReload if adopted, aerr := e.Adopted(); reload && (aerr != nil || !adopted) { reload = false diff --git a/internal/engine/expand.go b/internal/engine/expand.go index 0aa9c01..a6bcf88 100644 --- a/internal/engine/expand.go +++ b/internal/engine/expand.go @@ -222,6 +222,30 @@ func (x *expander) expand(req Request) ([]model.Op, error) { return []model.Op{x.versioned(req, o.Version, summary)}, nil case OpSettingsUpdate: return []model.Op{newOpRaw(OpSettingsUpdate, 0, 0, req.Data, "update settings")}, nil + case OpBlocklistCreate: + return x.blocklistCreate(req) + case OpBlocklistUpdate, OpBlocklistDelete: + b := x.st.Blocklists[req.ID] + if b == nil { + return nil, errf(KindNotFound, "blocklist %d does not exist", req.ID) + } + verb := "change" + if req.Op == OpBlocklistDelete { + verb = "delete" + } + return []model.Op{x.versioned(req, b.Version, verb+" blocklist "+b.Name)}, nil + case OpBlockRuleCreate: + return x.blockRuleCreate(req) + case OpBlockRuleUpdate, OpBlockRuleDelete: + r := x.st.BlockRules[req.ID] + if r == nil { + return nil, errf(KindNotFound, "rule %d does not exist", req.ID) + } + verb := "change rule: " + if req.Op == OpBlockRuleDelete { + verb = "delete rule: " + } + return []model.Op{x.versioned(req, r.Version, verb+ruleLabel(r))}, nil } return nil, errf(KindBadRequest, "unknown operation %q", req.Op) } diff --git a/internal/engine/inputs.go b/internal/engine/inputs.go index 8bffb51..c6379ec 100644 --- a/internal/engine/inputs.go +++ b/internal/engine/inputs.go @@ -30,6 +30,13 @@ const ( OpOptionDelete = "option.delete" OpSettingsUpdate = "settings.update" + OpBlocklistCreate = "blocklist.create" + OpBlocklistUpdate = "blocklist.update" + OpBlocklistDelete = "blocklist.delete" + OpBlockRuleCreate = "blockrule.create" + OpBlockRuleUpdate = "blockrule.update" + OpBlockRuleDelete = "blockrule.delete" + // Requests only: disabling and enabling records, by ID or by match // (zone, name, type and optionally data), become record.update // operations. @@ -40,7 +47,8 @@ const ( // OpKinds lists all operation kinds. var OpKinds = []string{OpZoneCreate, OpZoneUpdate, OpZoneDelete, OpRecordCreate, OpRecordUpdate, OpRecordDelete, OpRRSetReplace, OpViewCreate, OpViewUpdate, OpViewDelete, OpViewReorder, OpACLCreate, OpACLUpdate, OpACLDelete, - OpTSIGCreate, OpTSIGUpdate, OpTSIGDelete, OpOptionCreate, OpOptionUpdate, OpOptionDelete, OpSettingsUpdate} + OpTSIGCreate, OpTSIGUpdate, OpTSIGDelete, OpOptionCreate, OpOptionUpdate, OpOptionDelete, OpSettingsUpdate, + OpBlocklistCreate, OpBlocklistUpdate, OpBlocklistDelete, OpBlockRuleCreate, OpBlockRuleUpdate, OpBlockRuleDelete} // Request is an operation as submitted by a client. Requests are expanded // into one or more stored operations (for example an A record with "ptr" @@ -213,3 +221,26 @@ type SettingsInput struct { type DeleteInput struct { Cascade bool `json:"cascade,omitempty"` } + +// BlocklistInput creates or updates a blocklist. Views are given by name +// (Views) or ID (ViewIDs); an empty list means every view that offers +// recursion. +type BlocklistInput struct { + Name *string `json:"name,omitempty"` + URL *string `json:"url,omitempty"` + Views *[]string `json:"views,omitempty"` + ViewIDs *[]int64 `json:"view_ids,omitempty"` + Disabled *bool `json:"disabled,omitempty"` + Comment *string `json:"comment,omitempty"` +} + +// BlockRuleInput creates or updates a block or allow rule. Views as for +// BlocklistInput. +type BlockRuleInput struct { + Name *string `json:"name,omitempty"` + Action *string `json:"action,omitempty"` // block (default) or allow + Exact *bool `json:"exact,omitempty"` // only the name, not the names below it + Views *[]string `json:"views,omitempty"` + ViewIDs *[]int64 `json:"view_ids,omitempty"` + Comment *string `json:"comment,omitempty"` +} diff --git a/internal/engine/ops.go b/internal/engine/ops.go index 76bb1cb..2c2076a 100644 --- a/internal/engine/ops.go +++ b/internal/engine/ops.go @@ -98,6 +98,14 @@ func applyOp(st *model.State, op model.Op, c *applyCtx) error { err = applyOptionDelete(st, op, c) case OpSettingsUpdate: err = applySettings(st, op, c) + case OpBlocklistCreate, OpBlocklistUpdate: + err = applyBlocklistPut(st, op, c) + case OpBlocklistDelete: + err = applyBlocklistDelete(st, op, c) + case OpBlockRuleCreate, OpBlockRuleUpdate: + err = applyBlockRulePut(st, op, c) + case OpBlockRuleDelete: + err = applyBlockRuleDelete(st, op, c) default: err = errf(KindBadRequest, "unknown operation %q", op.Kind) } @@ -831,6 +839,9 @@ func applyViewDelete(st *model.State, op model.Op, c *applyCtx) error { if err := decode(op.Data, &in); err != nil { return err } + if users := blockingUsers(st, v.ID); len(users) > 0 { + return errf(KindConflict, "view %s is used by %s; change them to other views first", v.Name, strings.Join(users, ", ")) + } zones := st.ZonesInView(v.ID) if len(zones) > 0 && !in.Cascade { var names []string diff --git a/internal/model/blocking.go b/internal/model/blocking.go new file mode 100644 index 0000000..d16a15c --- /dev/null +++ b/internal/model/blocking.go @@ -0,0 +1,136 @@ +package model + +import ( + "slices" + "sort" + "strings" + "time" +) + +// Blocklist is a list of domains to block, downloaded from URL and kept up +// to date by jarld (docs/blocking.md). +type Blocklist struct { + ID int64 `json:"id"` + Name string `json:"name"` // a DNS label: it names the list's policy zone + URL string `json:"url"` + // ViewIDs are the views the list applies to; empty means every view + // that offers recursion (see BlockingViews). + ViewIDs []int64 `json:"view_ids"` + Disabled bool `json:"disabled,omitempty"` + Comment string `json:"comment"` + Meta +} + +// Block rule actions. +const ( + BlockActionBlock = "block" + BlockActionAllow = "allow" +) + +// BlockRule blocks or allows a domain, and unless Exact the names below +// it. Rules take precedence over blocklists. +type BlockRule struct { + ID int64 `json:"id"` + Name string `json:"name"` // lowercase, without the final dot + Action string `json:"action"` // block or allow + Exact bool `json:"exact,omitempty"` + ViewIDs []int64 `json:"view_ids"` // as for Blocklist + Comment string `json:"comment"` + Meta +} + +// BlocklistStatus is what jarld knows about a list's downloads. It is not +// configuration: it is kept outside changesets. +type BlocklistStatus struct { + ID int64 `json:"id"` + URL string `json:"url"` // the URL last tried + ETag string `json:"-"` + LastModified string `json:"-"` + CheckedAt *time.Time `json:"checked_at,omitempty"` // last attempt + UpdatedAt *time.Time `json:"updated_at,omitempty"` // last time the content changed + Entries int `json:"entries"` // domains + Records int `json:"records"` // policy zone records + Skipped int `json:"skipped"` // lines that weren't usable + SHA256 string `json:"-"` + Error string `json:"error,omitempty"` // the last attempt failed +} + +// BlocklistByName finds a blocklist by name. +func (s *State) BlocklistByName(name string) *Blocklist { + for _, b := range s.Blocklists { + if strings.EqualFold(b.Name, name) { + return b + } + } + return nil +} + +// SortedBlocklists returns the blocklists ordered by name. +func (s *State) SortedBlocklists() []*Blocklist { + out := make([]*Blocklist, 0, len(s.Blocklists)) + for _, b := range s.Blocklists { + out = append(out, b) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// SortedBlockRules returns the rules ordered by domain, parents first. +func (s *State) SortedBlockRules() []*BlockRule { + out := make([]*BlockRule, 0, len(s.BlockRules)) + for _, r := range s.BlockRules { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Name != b.Name { + return CanonicalLess(a.Name+".", b.Name+".") + } + if a.Exact != b.Exact { + return !a.Exact + } + return a.ID < b.ID + }) + return out +} + +// RecursionOn reports whether a view (0: the server without views) offers +// recursion: its recursion option, else the server's, else BIND's default +// (yes). +func (s *State) RecursionOn(viewID int64) bool { + o := s.FindOption(ScopeView, viewID, "recursion") + if viewID == 0 || o == nil { + if g := s.FindOption(ScopeOptions, 0, "recursion"); o == nil { + o = g + } + } + if o == nil { + return true + } + switch strings.ToLower(strings.Trim(strings.TrimSpace(o.Value), `";`)) { + case "no", "false", "0": + return false + } + return true +} + +// BlockingViews returns the views a blocklist or rule with viewIDs applies +// to, in view order: the given views, or when none are given every view +// that offers recursion (blocking only affects recursive queries). Without +// views it returns []int64{0}, the whole server, if it offers recursion. +func (s *State) BlockingViews(viewIDs []int64) []int64 { + views := s.OrderedViews() + if len(views) == 0 { + if !s.RecursionOn(0) { + return nil + } + return []int64{0} + } + var out []int64 + for _, v := range views { + if len(viewIDs) == 0 && s.RecursionOn(v.ID) || slices.Contains(viewIDs, v.ID) { + out = append(out, v.ID) + } + } + return out +} diff --git a/internal/model/model.go b/internal/model/model.go index 03043bc..c1c16c0 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -232,6 +232,9 @@ type State struct { Zones map[int64]*Zone Records map[int64]*Record Options map[int64]*Option + // Blocking (docs/blocking.md). + Blocklists map[int64]*Blocklist + BlockRules map[int64]*BlockRule } // NewState returns an empty state with default settings. @@ -244,6 +247,9 @@ func NewState() *State { Zones: map[int64]*Zone{}, Records: map[int64]*Record{}, Options: map[int64]*Option{}, + + Blocklists: map[int64]*Blocklist{}, + BlockRules: map[int64]*BlockRule{}, } } @@ -257,6 +263,9 @@ func (s *State) Clone() *State { Zones: make(map[int64]*Zone, len(s.Zones)), Records: make(map[int64]*Record, len(s.Records)), Options: make(map[int64]*Option, len(s.Options)), + + Blocklists: make(map[int64]*Blocklist, len(s.Blocklists)), + BlockRules: make(map[int64]*BlockRule, len(s.BlockRules)), } c.Settings.Nameservers = append([]string(nil), s.Settings.Nameservers...) for id, v := range s.Views { @@ -294,6 +303,16 @@ func (s *State) Clone() *State { n := *o c.Options[id] = &n } + for id, b := range s.Blocklists { + n := *b + n.ViewIDs = append([]int64(nil), b.ViewIDs...) + c.Blocklists[id] = &n + } + for id, r := range s.BlockRules { + n := *r + n.ViewIDs = append([]int64(nil), r.ViewIDs...) + c.BlockRules[id] = &n + } return c } diff --git a/internal/render/blocking.go b/internal/render/blocking.go new file mode 100644 index 0000000..3d464f9 --- /dev/null +++ b/internal/render/blocking.go @@ -0,0 +1,155 @@ +package render + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + "jarl/internal/blocking" + "jarl/internal/model" + "jarl/internal/namedconf" +) + +// KindPolicy is the kind of a rendered rules policy zone file. +const KindPolicy = "policy" + +// PolicyZone is a response policy zone in named's configuration +// (docs/blocking.md). +type PolicyZone struct { + Name string // without the final dot + ViewID int64 // 0 without views + View string // "" without views + File string + ListID int64 // the blocklist; 0 for the rules zone +} + +// BlocklistsDir holds the zone files of downloaded blocklists, one per +// list, shared by the views that use it. +func (p Paths) BlocklistsDir() string { return filepath.Join(p.CacheDir, "blocklists") } + +// BlocklistFile returns the zone file of a blocklist. It is named after +// the list's ID, so renaming the list doesn't move it. +func (p Paths) BlocklistFile(id int64) string { + return filepath.Join(p.BlocklistsDir(), fmt.Sprintf("list-%d.rpz", id)) +} + +// rulesFile returns a view's rules zone file. +func (p Paths) rulesFile(view string) string { + return filepath.Join(p.ZoneDir, ViewDir(view), blocking.RulesZone+".rpz") +} + +// PolicyZones returns the policy zones named loads, per view in view +// order, each view's in the order named checks them: the rules zone first +// (rules win over lists), then the enabled lists by name. +func PolicyZones(s *model.State, p Paths) []PolicyZone { + var out []PolicyZone + rules, lists := s.SortedBlockRules(), s.SortedBlocklists() + scopes := []int64{0} + if len(s.Views) > 0 { + scopes = nil + for _, v := range s.OrderedViews() { + scopes = append(scopes, v.ID) + } + } + for _, vid := range scopes { + view := "" + if v := s.Views[vid]; v != nil { + view = v.Name + } + for _, r := range rules { + if slices.Contains(s.BlockingViews(r.ViewIDs), vid) { + out = append(out, PolicyZone{Name: blocking.RulesZone, ViewID: vid, View: view, File: p.rulesFile(view)}) + break + } + } + for _, b := range lists { + if !b.Disabled && slices.Contains(s.BlockingViews(b.ViewIDs), vid) { + out = append(out, PolicyZone{Name: blocking.ListZone(b.Name), ViewID: vid, View: view, File: p.BlocklistFile(b.ID), ListID: b.ID}) + } + } + } + return out +} + +// ViewRules returns the rules that apply in a view (0 without views). +func ViewRules(s *model.State, viewID int64) []blocking.Rule { + var out []blocking.Rule + for _, r := range s.SortedBlockRules() { + if slices.Contains(s.BlockingViews(r.ViewIDs), viewID) { + out = append(out, blocking.Rule{Name: r.Name, Action: r.Action, Exact: r.Exact}) + } + } + return out +} + +// writeResponsePolicy writes the response-policy statement for a view's +// policy zones (or the server's, without views). Rewrites are logged to +// Jarl's response log when statistics read it, and not at all otherwise +// (they would go to named's default log). +func writeResponsePolicy(w *confWriter, zones []PolicyZone, logRewrites bool) { + if len(zones) == 0 { + return + } + w.begin("blocking", 0, "blocking (rules and blocklists)") + w.write("\t// Blocking (docs/blocking.md): rules first, then blocklists.\n\tresponse-policy {\n") + for _, z := range zones { + // Apply a new version at once: Jarl reloads a list only when it + // changed (at most every blocking.refresh), so named's default + // delay for batching frequent updates only adds a wait. + opts := []string{"min-update-interval 0"} + if !logRewrites { + opts = append(opts, "log no") + } + w.write(fmt.Sprintf("\t\tzone %s%s;\n", namedconf.QuoteString(z.Name), prefixSpace(strings.Join(opts, " ")))) + } + // Block without waiting for the real answer, and also for clients + // that ask for DNSSEC data. + w.write("\t} qname-wait-recurse no break-dnssec yes;\n") + w.end() +} + +func prefixSpace(s string) string { + if s == "" { + return "" + } + return " " + s +} + +// writePolicyZones writes the zone statements of policy zones and adds +// their files to the output: rules zone files are rendered, blocklist +// files only listed (jarld downloads them; see Output.Blocklists). +func writePolicyZones(w *confWriter, s *model.State, zones []PolicyZone, out *Output, indent int) { + tabs := strings.Repeat("\t", indent) + for _, z := range zones { + w.write("\n") + label := "blocking: your rules" + if z.ListID != 0 { + label = "blocking: blocklist " + s.Blocklists[z.ListID].Name + } + w.begin("blocking", z.ListID, label) + w.write(fmt.Sprintf("%szone %s {\n%s\ttype primary;\n%s\tfile %s;\n%s\tallow-query { none; };\n%s\tallow-transfer { none; };\n%s\tnotify no;\n%s};\n", + tabs, namedconf.QuoteString(z.Name), tabs, tabs, namedconf.QuoteString(z.File), tabs, tabs, tabs, tabs)) + w.end() + if z.ListID != 0 { + out.Blocklists = appendUnique(out.Blocklists, z.File) + continue + } + title := "Blocking rules" + if z.View != "" { + title += " for view " + z.View + } + out.Files = append(out.Files, &File{Path: z.File, Content: []byte(blocking.RulesZoneFile(title, ViewRules(s, z.ViewID))), Mode: 0o640, Kind: KindPolicy}) + } +} + +// scopeZones returns the policy zones of one view (0 without views). +func scopeZones(all []PolicyZone, viewID int64) []PolicyZone { + var out []PolicyZone + for _, z := range all { + if z.ViewID == viewID { + out = append(out, z) + } + } + return out +} diff --git a/internal/render/render.go b/internal/render/render.go index e3fab7f..41dc819 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -69,6 +69,11 @@ type Output struct { Spans map[string][]Span // ZoneFiles maps zone IDs to the file named loads for the zone. ZoneFiles map[int64]string + // Policy lists the response policy zones (docs/blocking.md). + Policy []PolicyZone + // Blocklists are the zone files of blocklists named loads. jarld + // downloads them; until then they must exist, as empty policy zones. + Blocklists []string } // File returns the rendered file at path, or nil. @@ -264,6 +269,9 @@ func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Spa st := x.Stats logResponses := st != nil && st.LogFile != "" + pzs := PolicyZones(s, p) + out.Policy = pzs + logRewrites := logResponses && len(pzs) > 0 sawStats, sawLogging := false, false if top := s.ScopedOptions(model.ScopeTopLevel, 0); len(top) > 0 { w.write("\n") @@ -274,7 +282,7 @@ func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Spa w.write(st.statsStatement(o)) sawStats = true case logResponses && o.Name == "logging" && !sawLogging: - w.write(st.loggingStatement(o)) + w.write(st.loggingStatement(o, logRewrites)) sawLogging = true default: w.write(optionText(o, 0)) @@ -289,7 +297,7 @@ func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Spa w.write(st.statsStatement(nil)) } if logResponses && !sawLogging { - w.write(st.loggingStatement(nil)) + w.write(st.loggingStatement(nil, logRewrites)) } w.end() } @@ -322,10 +330,14 @@ func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Spa w.write("\t// " + statsComment + "\n\tresponselog yes;\n") w.end() } - w.write("};\n") - views := s.OrderedViews() if len(views) == 0 { + writeResponsePolicy(w, scopeZones(pzs, 0), logResponses) + } + w.write("};\n") + + if len(views) == 0 { + writePolicyZones(w, s, scopeZones(pzs, 0), out, 0) for _, z := range s.ZonesInView(0) { w.write("\n") renderZone(w, s, z, p, out, 0) @@ -352,6 +364,8 @@ func renderConfig(s *model.State, p Paths, x Extras, out *Output) (string, []Spa w.write(optionText(o, 1)) w.end() } + writeResponsePolicy(w, scopeZones(pzs, v.ID), logResponses) + writePolicyZones(w, s, scopeZones(pzs, v.ID), out, 1) for _, z := range s.ZonesInView(v.ID) { w.write("\n") renderZone(w, s, z, p, out, 1) diff --git a/internal/render/stats.go b/internal/render/stats.go index 769aa8f..8da6c44 100644 --- a/internal/render/stats.go +++ b/internal/render/stats.go @@ -69,8 +69,9 @@ func (h *StatsHooks) statsStatement(user *model.Option) string { } // loggingStatement returns the logging statement with Jarl's channel and -// the responses category, merged into the user's own if there is one. -func (h *StatsHooks) loggingStatement(user *model.Option) string { +// the responses category (and rpz, for blocking's rewrites), merged into +// the user's own if there is one. +func (h *StatsHooks) loggingStatement(user *model.Option, rpz bool) string { var st *namedconf.Stmt if user != nil { var err error @@ -84,15 +85,21 @@ func (h *StatsHooks) loggingStatement(user *model.Option) string { b := st.FirstBlock() // Channels must be defined before the categories that use them. b.Stmts = append([]*namedconf.Stmt{h.logChannel()}, b.Stmts...) - merged := false - for _, s := range b.Stmts { - if s.Keyword() == "category" && strings.EqualFold(s.Arg(0), "responses") && s.FirstBlock() != nil { - s.FirstBlock().Stmts = append(s.FirstBlock().Stmts, namedconf.NewStmt(StatsChannelName)) - merged = true - } + categories := []string{"responses"} + if rpz { + categories = append(categories, "rpz") } - if !merged { - b.Stmts = append(b.Stmts, namedconf.NewStmt("category", "responses", namedconf.NewBlock(namedconf.NewStmt(StatsChannelName)))) + for _, cat := range categories { + merged := false + for _, s := range b.Stmts { + if s.Keyword() == "category" && strings.EqualFold(s.Arg(0), cat) && s.FirstBlock() != nil { + s.FirstBlock().Stmts = append(s.FirstBlock().Stmts, namedconf.NewStmt(StatsChannelName)) + merged = true + } + } + if !merged { + b.Stmts = append(b.Stmts, namedconf.NewStmt("category", cat, namedconf.NewBlock(namedconf.NewStmt(StatsChannelName)))) + } } prefix := "" if user != nil { diff --git a/internal/server/blocking_test.go b/internal/server/blocking_test.go new file mode 100644 index 0000000..2535e02 --- /dev/null +++ b/internal/server/blocking_test.go @@ -0,0 +1,95 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestBlockingAPI(t *testing.T) { + e := newAPIEnv(t) + lists := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/missing" { + http.NotFound(w, r) + return + } + w.Write([]byte("||ads.example^\n0.0.0.0 tracker.example\n")) + })) + defer lists.Close() + + b := e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": lists.URL + "/list.txt"}, 201)["object"].(map[string]any) + if b["name"] != "ads" || b["state"] != "waiting" { + t.Errorf("new list = %v", b) + } + got := e.do("POST", "/blocking/lists/ads/refresh", e.token, nil, 200) + st, _ := got["status"].(map[string]any) + if got["state"] != "ok" || st["entries"] != 2.0 || st["records"] != 3.0 { + t.Errorf("after refresh = %v", got) + } + e.do("POST", "/blocking/rules", e.token, map[string]any{"name": "Good.Ads.Example.", "action": "allow"}, 201) + r := e.do("POST", "/blocking/rules", e.token, map[string]any{"name": "more.example", "exact": true}, 201)["object"].(map[string]any) + if r["name"] != "more.example" || r["action"] != "block" || r["exact"] != true { + t.Errorf("new rule = %v", r) + } + for name, want := range map[string]string{"x.ads.example": "blocked", "a.good.ads.example": "allowed", "more.example": "blocked", "sub.more.example": "none"} { + m := e.list("/blocking/check?name="+name, e.token) + if len(m) != 1 || m[0]["result"] != want { + t.Errorf("check %s = %v, want %s", name, m, want) + } + } + if _, status := e.req("GET", "/blocking/check?name=com", e.token, nil); status != 400 { + t.Errorf("check of a TLD = %d", status) + } + + // Staged changes show as pending. + cs := itoa(int64(e.do("POST", "/changesets", e.token, map[string]any{"description": "more"}, 201)["id"].(float64))) + e.do("POST", "/blocking/rules?changeset="+cs, e.token, map[string]any{"name": "staged.example"}, 202) + e.do("PATCH", "/blocking/lists/ads?changeset="+cs, e.token, map[string]any{"disabled": true}, 202) + pending := map[string]any{} + for _, x := range e.list("/blocking/rules?changeset="+cs, e.token) { + pending[x["name"].(string)] = x["pending"] + } + if pending["staged.example"] != "add" || pending["more.example"] != nil { + t.Errorf("pending rules = %v", pending) + } + if l := e.list("/blocking/lists?changeset="+cs, e.token); len(l) != 1 || l[0]["pending"] != "change" || l[0]["state"] != "disabled" { + t.Errorf("pending list = %v", l) + } + e.do("POST", "/changesets/"+cs+"/discard", e.token, nil, 204) + + // Delete rules by name; a missing one is a 404 unless missing_ok. + e.do("DELETE", "/blocking/rules?name=more.example", e.token, nil, 200) + if _, status := e.req("DELETE", "/blocking/rules?name=more.example", e.token, nil); status != 404 { + t.Errorf("deleting a missing rule = %d", status) + } + e.do("DELETE", "/blocking/rules?name=more.example&missing_ok=1", e.token, nil, 200) + + // Readers see blocking but can't change it. + e.do("POST", "/users", e.token, map[string]any{"username": "reader", "password": "reader-password-123", "grant_strings": []string{"blocking:read"}}, 201) + var readerID int64 + for _, u := range e.list("/users", e.token) { + if u["username"] == "reader" { + readerID = int64(u["id"].(float64)) + } + } + rt := e.do("POST", "/users/"+itoa(readerID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string) + if l := e.list("/blocking/lists", rt); len(l) != 1 { + t.Errorf("reader's lists = %v", l) + } + for _, c := range []struct{ method, path string }{{"POST", "/blocking/rules"}, {"POST", "/blocking/lists/ads/refresh"}, {"DELETE", "/blocking/lists/ads"}} { + if _, status := e.req(c.method, c.path, rt, map[string]any{"name": "x.example"}); status != 403 { + t.Errorf("reader %s %s = %d", c.method, c.path, status) + } + } + // A list that can't be downloaded shows on the dashboard. + e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "gone", "url": lists.URL + "/missing"}, 201) + if l := e.do("POST", "/blocking/lists/gone/refresh", e.token, nil, 200); l["state"] != "error" { + t.Errorf("failed list = %v", l) + } + errs, _ := e.do("GET", "/status", e.token, nil, 200)["blocklist_errors"].([]any) + if len(errs) != 1 || errs[0].(map[string]any)["name"] != "gone" { + t.Errorf("status blocklist_errors = %v", errs) + } + e.do("DELETE", "/blocking/lists/gone", e.token, nil, 200) + e.do("DELETE", "/blocking/lists/ads", e.token, nil, 200) +} diff --git a/internal/server/h_blocking.go b/internal/server/h_blocking.go new file mode 100644 index 0000000..20215d9 --- /dev/null +++ b/internal/server/h_blocking.go @@ -0,0 +1,368 @@ +package server + +import ( + "context" + "net/http" + "slices" + "strconv" + "strings" + "time" + + "jarl/internal/api" + "jarl/internal/auth" + "jarl/internal/blocking" + "jarl/internal/engine" + "jarl/internal/model" +) + +// ---- blocking (docs/blocking.md) + +// bytesPerRecord is what named 9.20 needs per policy zone record and view, +// measured with lists of a few hundred thousand entries. +const bytesPerRecord = 630 + +func viewNames(st *model.State, ids []int64) []string { + out := []string{} + for _, id := range ids { + if v := st.Views[id]; v != nil { + out = append(out, v.Name) + } + } + return out +} + +func presentBlocklist(st *model.State, b *model.Blocklist, status *model.BlocklistStatus) api.Blocklist { + views := st.BlockingViews(b.ViewIDs) + out := api.Blocklist{Blocklist: *b, Views: viewNames(st, views), AutoViews: len(b.ViewIDs) == 0 && len(st.Views) > 0, Status: status} + switch { + case b.Disabled: + out.State = "disabled" + case len(views) == 0: + out.State = "idle" + case status != nil && status.Error != "": + out.State = "error" + case status == nil || status.UpdatedAt == nil: + out.State = "waiting" + default: + out.State = "ok" + } + if status != nil && !b.Disabled { + out.MemoryMB = int((int64(status.Records)*bytesPerRecord*int64(len(views)) + 1<<19) >> 20) + } + return out +} + +func presentBlockRule(st *model.State, r *model.BlockRule) api.BlockRule { + return api.BlockRule{BlockRule: *r, Views: viewNames(st, st.BlockingViews(r.ViewIDs)), AutoViews: len(r.ViewIDs) == 0 && len(st.Views) > 0} +} + +func requireBlocking(p *auth.Principal) error { + return require(p, auth.Action{Resource: auth.ResBlocking}) +} + +func (s *Server) listBlocklists(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := requireBlocking(p); err != nil { + return err + } + st, base, err := s.states(r) + if err != nil { + return err + } + statuses, err := s.store.BlocklistStatuses(r.Context()) + if err != nil { + return err + } + out := []api.Blocklist{} + for _, b := range st.SortedBlocklists() { + o := presentBlocklist(st, b, statuses[b.ID]) + o.Pending = pending(base, base != nil && base.Blocklists[b.ID] == nil, base != nil && base.Blocklists[b.ID] != nil && base.Blocklists[b.ID].Version != b.Version) + out = append(out, o) + } + if base != nil { + for _, b := range base.SortedBlocklists() { + if st.Blocklists[b.ID] == nil { + o := presentBlocklist(base, b, statuses[b.ID]) + o.Pending = "delete" + out = append(out, o) + } + } + } + writeJSON(w, http.StatusOK, out) + return nil +} + +// pending returns the pending marker of an object listed with a changeset. +func pending(base *model.State, added, changed bool) string { + switch { + case base == nil: + return "" + case added: + return "add" + case changed: + return "change" + } + return "" +} + +func (s *Server) blocklistByPath(r *http.Request, st *model.State) (*model.Blocklist, error) { + ref := r.PathValue("id") + var b *model.Blocklist + if id, err := strconv.ParseInt(ref, 10, 64); err == nil { + b = st.Blocklists[id] + } else { + b = st.BlocklistByName(ref) + } + if b == nil { + return nil, notFound("blocklist %s not found", ref) + } + return b, nil +} + +func (s *Server) blocklistObject(ctx context.Context) func(*model.State, int64) any { + return func(st *model.State, id int64) any { + b := st.Blocklists[id] + if b == nil { + return nil + } + statuses, _ := s.store.BlocklistStatuses(ctx) + return presentBlocklist(st, b, statuses[id]) + } +} + +func (s *Server) getBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := requireBlocking(p); err != nil { + return err + } + st, err := s.state(r) + if err != nil { + return err + } + b, err := s.blocklistByPath(r, st) + if err != nil { + return err + } + writeJSON(w, http.StatusOK, s.blocklistObject(r.Context())(st, b.ID)) + return nil +} + +func (s *Server) createBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + body, err := readBody(r) + if err != nil { + return err + } + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistCreate, 0, body)}, s.blocklistObject(r.Context())) +} + +func (s *Server) updateBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + st, err := s.state(r) + if err != nil { + return err + } + b, err := s.blocklistByPath(r, st) + if err != nil { + return err + } + body, err := readBody(r) + if err != nil { + return err + } + id := b.ID + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistUpdate, id, body)}, func(st *model.State, _ int64) any { return s.blocklistObject(r.Context())(st, id) }) +} + +func (s *Server) deleteBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + st, err := s.state(r) + if err != nil { + return err + } + b, err := s.blocklistByPath(r, st) + if err != nil { + return err + } + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlocklistDelete, b.ID, nil)}, nil) +} + +// refreshBlocklist: POST /blocking/lists/{id}/refresh[?force=1] downloads +// the list now and waits for the result. +func (s *Server) refreshBlocklist(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := require(p, auth.Action{Resource: auth.ResBlocking, Write: true}); err != nil { + return err + } + st, err := s.store.LoadState(r.Context()) + if err != nil { + return err + } + b, err := s.blocklistByPath(r, st) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute) + defer cancel() + status, rerr := s.eng.RefreshBlocklist(ctx, b.ID, queryBool(r, "force")) + if status == nil { + return rerr + } + writeJSON(w, http.StatusOK, presentBlocklist(st, b, status)) + return nil +} + +func (s *Server) listBlockRules(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := requireBlocking(p); err != nil { + return err + } + st, base, err := s.states(r) + if err != nil { + return err + } + out := []api.BlockRule{} + for _, rule := range st.SortedBlockRules() { + o := presentBlockRule(st, rule) + o.Pending = pending(base, base != nil && base.BlockRules[rule.ID] == nil, base != nil && base.BlockRules[rule.ID] != nil && base.BlockRules[rule.ID].Version != rule.Version) + out = append(out, o) + } + if base != nil { + for _, rule := range base.SortedBlockRules() { + if st.BlockRules[rule.ID] == nil { + o := presentBlockRule(base, rule) + o.Pending = "delete" + out = append(out, o) + } + } + } + writeJSON(w, http.StatusOK, out) + return nil +} + +func ruleObject(st *model.State, id int64) any { + if rule := st.BlockRules[id]; rule != nil { + return presentBlockRule(st, rule) + } + return nil +} + +func (s *Server) ruleByPath(r *http.Request, st *model.State) (*model.BlockRule, error) { + id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64) + rule := st.BlockRules[id] + if rule == nil { + return nil, notFound("rule %s not found", r.PathValue("id")) + } + return rule, nil +} + +func (s *Server) createBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + body, err := readBody(r) + if err != nil { + return err + } + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleCreate, 0, body)}, ruleObject) +} + +func (s *Server) updateBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + st, err := s.state(r) + if err != nil { + return err + } + rule, err := s.ruleByPath(r, st) + if err != nil { + return err + } + body, err := readBody(r) + if err != nil { + return err + } + id := rule.ID + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleUpdate, id, body)}, func(st *model.State, _ int64) any { return ruleObject(st, id) }) +} + +func (s *Server) deleteBlockRule(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + st, err := s.state(r) + if err != nil { + return err + } + rule, err := s.ruleByPath(r, st) + if err != nil { + return err + } + return s.mutate(w, r, p, "", []engine.Request{request(engine.OpBlockRuleDelete, rule.ID, nil)}, nil) +} + +// deleteBlockRulesByName: DELETE /blocking/rules?name=NAME[&view=VIEW] +// deletes the rules for a domain (in one view, or all). +func (s *Server) deleteBlockRulesByName(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + st, err := s.state(r) + if err != nil { + return err + } + q := r.URL.Query() + name := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(q.Get("name"))), ".") + if name == "" { + return badRequest("give the domain of the rules to delete (name=)") + } + var viewID int64 = -1 + if vn := q.Get("view"); vn != "" { + v := st.ViewByName(vn) + if v == nil { + return notFound("view %s not found", vn) + } + viewID = v.ID + } + var reqs []engine.Request + for _, rule := range st.SortedBlockRules() { + if rule.Name != name { + continue + } + if viewID >= 0 && !slices.Contains(st.BlockingViews(rule.ViewIDs), viewID) { + continue + } + reqs = append(reqs, request(engine.OpBlockRuleDelete, rule.ID, nil)) + } + if len(reqs) == 0 { + if queryBool(r, "missing_ok") { + writeJSON(w, http.StatusOK, api.MutationResponse{}) + return nil + } + return notFound("there is no rule for %s", name) + } + return s.mutate(w, r, p, "", reqs, nil) +} + +// checkBlocking: GET /blocking/check?name=NAME says what blocking does +// with a name in each view. +func (s *Server) checkBlocking(w http.ResponseWriter, r *http.Request, p *auth.Principal) error { + if err := requireBlocking(p); err != nil { + return err + } + st, err := s.store.LoadState(r.Context()) + if err != nil { + return err + } + out, err := s.eng.CheckBlocking(st, r.URL.Query().Get("name")) + if err != nil { + return err + } + if out == nil { + out = []blocking.Match{} + } + writeJSON(w, http.StatusOK, out) + return nil +} + +// blocklistErrors lists the enabled blocklists whose last update failed, +// for the dashboard. +func (s *Server) blocklistErrors(ctx context.Context) []api.BlocklistError { + st, err := s.store.LoadState(ctx) + if err != nil || len(st.Blocklists) == 0 { + return nil + } + statuses, err := s.store.BlocklistStatuses(ctx) + if err != nil { + return nil + } + var out []api.BlocklistError + for _, b := range st.SortedBlocklists() { + if s := statuses[b.ID]; s != nil && s.Error != "" && !b.Disabled { + out = append(out, api.BlocklistError{Name: b.Name, Error: s.Error, UpdatedAt: s.UpdatedAt}) + } + } + return out +} diff --git a/internal/server/h_changes.go b/internal/server/h_changes.go index 712739d..bf15bc6 100644 --- a/internal/server/h_changes.go +++ b/internal/server/h_changes.go @@ -279,6 +279,9 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa resp.Backup = s.backupStatus(r.Context(), now) resp.TOTP = s.totpOverview(r.Context()) } + if p.Can(auth.Action{Resource: auth.ResBlocking}) { + resp.BlocklistErrors = s.blocklistErrors(r.Context()) + } writeJSON(w, http.StatusOK, resp) return nil } diff --git a/internal/server/routes.go b/internal/server/routes.go index 9327370..418ffbf 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -56,6 +56,19 @@ func (s *Server) routes() { m.HandleFunc("DELETE "+p+"/views/{id}", s.authed(s.deleteView)) m.HandleFunc("PUT "+p+"/views/{id}/color", s.authed(s.setViewColor)) // ACLs + // Blocking + m.HandleFunc("GET "+p+"/blocking/lists", s.authed(s.listBlocklists)) + m.HandleFunc("POST "+p+"/blocking/lists", s.authed(s.createBlocklist)) + m.HandleFunc("GET "+p+"/blocking/lists/{id}", s.authed(s.getBlocklist)) + m.HandleFunc("PATCH "+p+"/blocking/lists/{id}", s.authed(s.updateBlocklist)) + m.HandleFunc("DELETE "+p+"/blocking/lists/{id}", s.authed(s.deleteBlocklist)) + m.HandleFunc("POST "+p+"/blocking/lists/{id}/refresh", s.authed(s.refreshBlocklist)) + m.HandleFunc("GET "+p+"/blocking/rules", s.authed(s.listBlockRules)) + m.HandleFunc("POST "+p+"/blocking/rules", s.authed(s.createBlockRule)) + m.HandleFunc("DELETE "+p+"/blocking/rules", s.authed(s.deleteBlockRulesByName)) + m.HandleFunc("PATCH "+p+"/blocking/rules/{id}", s.authed(s.updateBlockRule)) + m.HandleFunc("DELETE "+p+"/blocking/rules/{id}", s.authed(s.deleteBlockRule)) + m.HandleFunc("GET "+p+"/blocking/check", s.authed(s.checkBlocking)) m.HandleFunc("GET "+p+"/acls", s.authed(s.listACLs)) m.HandleFunc("POST "+p+"/acls", s.authed(s.createACL)) m.HandleFunc("GET "+p+"/acls/{id}", s.authed(s.getACL)) diff --git a/internal/stats/blocking_test.go b/internal/stats/blocking_test.go new file mode 100644 index 0000000..a6ec81a --- /dev/null +++ b/internal/stats/blocking_test.go @@ -0,0 +1,111 @@ +package stats + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/netip" + "os" + "path/filepath" + "testing" + "time" +) + +func TestParseRewrite(t *testing.T) { + loc := time.UTC + // Real lines from named 9.20.24. + rw, ok := ParseRewrite(`09-Oct-2026 23:49:13.089 client @0x7bd607320000 127.0.0.1#56905 (x.ads.example): view internal: rpz QNAME NXDOMAIN rewrite x.ads.example/A/IN via x.ads.example.rules.jarl.invalid`, loc) + if !ok || rw.Client.String() != "127.0.0.1" || rw.Port != "56905" || rw.View != "internal" || rw.Name != "x.ads.example" || rw.Action != "NXDOMAIN" || policySource(rw.Via) != "rules" { + t.Errorf("rules rewrite: %+v (%s)", rw, policySource(rw.Via)) + } + rw, ok = ParseRewrite(`09-Oct-2026 23:49:13.378 client @0x7bd607320000 10.0.0.9#57347 (Tracker.Example): rpz QNAME NXDOMAIN rewrite tracker.example/AAAA/IN via tracker.example.hagezi-pro.blocklist.jarl.invalid`, loc) + if !ok || rw.View != "" || rw.Name != "tracker.example" || policySource(rw.Via) != "hagezi-pro" { + t.Errorf("list rewrite: %+v (%s)", rw, policySource(rw.Via)) + } + if policySource("ads.example.my-own-rpz.") != "ads.example.my-own-rpz" { + t.Error("a policy zone that isn't Jarl's") + } + if _, ok := ParseRewrite(`09-Oct-2026 23:49:13.378 client @0x7bd607320000 10.0.0.9#57347 (x): view internal: response: x IN A NOERROR 1 0 1 + (10.0.0.2)`, loc); ok { + t.Error("a response line parsed as a rewrite") + } +} + +func TestCollectorBlocking(t *testing.T) { + ctx := context.Background() + loc := time.UTC + dir := t.TempDir() + logFile := filepath.Join(dir, "responses.log") + store, err := OpenStore(filepath.Join(dir, "stats.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + named := newFakeNamed(t) + clk := &clock{time.Date(2026, 10, 5, 10, 0, 30, 0, loc)} + opts := Options{QueryLog: true, LogFile: logFile, StatisticsChannel: "unused", TopN: 5, + RetainMinute: 7 * 24 * time.Hour, RetainHour: 90 * 24 * time.Hour, RetainDay: 730 * 24 * time.Hour, + Exclude: []netip.Prefix{netip.MustParsePrefix("10.0.0.250/32")}, Location: loc} + c := NewCollector(store, opts, func() map[string][]string { return nil }, slog.New(slog.NewTextHandler(io.Discard, nil))) + c.poller.URL = named.srv.URL + c.now = clk.now + c.start(ctx) + + at := time.Date(2026, 10, 5, 10, 0, 40, 0, loc).Format(logTimeLayout) + rewrite := func(client, port, name, action, via string) string { + return fmt.Sprintf("%s client @0x7f0000000000 %s#%s (%s): view internal: rpz QNAME %s rewrite %s/A/IN via %s\n", at, client, port, name, action, name, via) + } + response := func(client, port, name, rcode string) string { + return fmt.Sprintf("%s client @0x7f0000000000 %s#%s (%s): view internal: response: %s IN A %s 0 0 2 +E(0)K (10.0.0.2)\n", at, client, port, name, name, rcode) + } + f, _ := os.Create(logFile) + for _, l := range []string{ + rewrite("10.0.0.5", "1001", "ads.example", "NXDOMAIN", "ads.example.hagezi.blocklist.jarl.invalid"), + response("10.0.0.5", "1001", "ads.example", "NXDOMAIN"), + rewrite("10.0.0.5", "1002", "x.ads.example", "NXDOMAIN", "x.ads.example.hagezi.blocklist.jarl.invalid"), + response("10.0.0.5", "1002", "x.ads.example", "NXDOMAIN"), + rewrite("10.0.0.6", "1003", "ads.example", "NXDOMAIN", "ads.example.rules.jarl.invalid"), + response("10.0.0.6", "1003", "ads.example", "NXDOMAIN"), + // An allow rule: answered normally, not blocked. + rewrite("10.0.0.6", "1004", "cdn.ads.example", "PASSTHRU", "cdn.ads.example.rules.jarl.invalid"), + response("10.0.0.6", "1004", "cdn.ads.example", "NOERROR"), + // A real NXDOMAIN. + response("10.0.0.6", "1005", "typo.example", "NXDOMAIN"), + // An excluded client. + rewrite("10.0.0.250", "1006", "ads.example", "NXDOMAIN", "ads.example.hagezi.blocklist.jarl.invalid"), + response("10.0.0.250", "1006", "ads.example", "NXDOMAIN"), + } { + f.WriteString(l) + } + f.Close() + c.readLog() + named.add(6, 4) + clk.t = time.Date(2026, 10, 5, 10, 1, 2, 0, loc) + c.flush(ctx, false) + + m := time.Date(2026, 10, 5, 10, 0, 0, 0, loc).Unix() + got, _ := store.sums(ctx, ResMinute, m, m+60, "") + for k, v := range map[string]int64{"blocked": 3, "blocklist:hagezi": 2, "blocklist:rules": 1, "responses": 5} { + if got[k] != v { + t.Errorf("%s = %d, want %d (all: %v)", k, got[k], v, got) + } + } + if b, _ := store.TopsAt(ctx, ResHour, m, KindBlocked); len(b) != 2 || b[0] != (Entry{Key: "ads.example", Count: 2}) { + t.Errorf("blocked names = %v", b) + } + if nx, _ := store.TopsAt(ctx, ResHour, m, KindNXDomain); len(nx) != 1 || nx[0].Key != "typo.example" { + t.Errorf("NXDOMAIN names should leave out blocked ones: %v", nx) + } + q := &Querier{Store: store, Opts: opts, Now: clk.now} + sum, err := q.Summary(ctx, time.Hour, 5) + if err != nil { + t.Fatal(err) + } + // 6 queries, 4 NXDOMAIN of which 3 blocked. + if sum.Blocked != 3 || sum.BlockedRatio == nil || *sum.BlockedRatio != 0.5 || sum.NXDomainRatio == nil || *sum.NXDomainRatio != 1.0/6 { + t.Errorf("summary: blocked %d ratio %v nxdomain %v", sum.Blocked, sum.BlockedRatio, sum.NXDomainRatio) + } + if len(sum.Blocklists) != 2 || sum.Blocklists[0] != (Entry{Key: "hagezi", Count: 2}) || len(sum.Tops[KindBlocked]) != 2 { + t.Errorf("summary lists %v, tops %v", sum.Blocklists, sum.Tops[KindBlocked]) + } +} diff --git a/internal/stats/collector.go b/internal/stats/collector.go index 666bc8b..e00744a 100644 --- a/internal/stats/collector.go +++ b/internal/stats/collector.go @@ -12,6 +12,8 @@ import ( "strings" "sync" "time" + + "jarl/internal/blocking" ) // Kinds of top lists. @@ -21,10 +23,11 @@ const ( KindNXDomain = "nxdomain" // names answered with NXDOMAIN KindServFail = "servfail" // names answered with SERVFAIL KindRefused = "refused" // clients that were refused + KindBlocked = "blocked" // names blocked (docs/blocking.md) ) // Kinds lists the top-list kinds. -var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused} +var Kinds = []string{KindName, KindClient, KindNXDomain, KindServFail, KindRefused, KindBlocked} // OtherZone is the zone key for names outside every local zone (answered // by recursion, or refused). @@ -99,6 +102,9 @@ type Collector struct { recentL []int64 // response log lines per minute, last 10 minutes linesMin int64 ensureAt time.Time + // rewritten holds queries named blocked (client#port|name) until their + // response line arrives. + rewritten map[string]bool } // maxKeys bounds the per-hour maps; rarely seen keys are dropped first. @@ -296,8 +302,15 @@ func (c *Collector) HandleLine(line string) { func (c *Collector) handleLine(line string) { ev, ok := ParseResponse(line, c.opts.Location) if !ok { + if rw, ok := ParseRewrite(line, c.opts.Location); ok { + c.handleRewrite(rw) + } return } + blocked := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name] + if blocked { + delete(c.rewritten, ev.Client.String()+"#"+ev.Port+"|"+ev.Name) + } c.status.LinesRead++ c.linesMin++ t := ev.Time @@ -324,12 +337,14 @@ func (c *Collector) handleLine(line string) { tops := hours[hour] tops[KindName][ev.Name]++ tops[KindClient][client]++ - switch ev.RCode { - case "NXDOMAIN": + switch { + case blocked: + // Counted by handleRewrite; not a name that doesn't exist. + case ev.RCode == "NXDOMAIN": tops[KindNXDomain][ev.Name]++ - case "SERVFAIL": + case ev.RCode == "SERVFAIL": tops[KindServFail][ev.Name]++ - case "REFUSED": + case ev.RCode == "REFUSED": tops[KindRefused][client]++ } for _, m := range tops { @@ -339,6 +354,48 @@ func (c *Collector) handleLine(line string) { } } +// handleRewrite counts a query that blocking answered: "blocked", and +// "blocklist:NAME" for the list (or "rules") that matched. +func (c *Collector) handleRewrite(rw Rewrite) { + if rw.Action == "PASSTHRU" { + return // an allow rule: answered normally + } + for _, p := range c.opts.Exclude { + if p.Contains(rw.Client) { + return + } + } + if c.rewritten == nil || len(c.rewritten) > 10_000 { + // Rewrites whose response line never came (named only logs + // responses while response logging is on) don't pile up. + c.rewritten = map[string]bool{} + } + c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true + minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time) + c.count(minute, "blocked", 1) + c.count(minute, "blocklist:"+policySource(rw.Via), 1) + hours := c.hours + if hour < c.liveHour { + hours = c.late + } + hours[hour] = c.hourMaps(hours, hour) + hours[hour][KindBlocked][rw.Name]++ +} + +// policySource names what a rewrite came from: a blocklist's name, "rules" +// for the rules entered in Jarl, or the policy zone for others. +func policySource(via string) string { + via = strings.TrimSuffix(via, ".") + if strings.HasSuffix(via, "."+blocking.RulesZone) { + return "rules" + } + if i := strings.LastIndex(via, ".blocklist.jarl.invalid"); i > 0 && i+len(".blocklist.jarl.invalid") == len(via) { + rest := via[:i] + return rest[strings.LastIndex(rest, ".")+1:] + } + return via +} + // shrink drops the rarest keys until the map is well under maxKeys. func shrink(m map[string]int64) { for limit := int64(1); len(m) > maxKeys*8/10; limit++ { diff --git a/internal/stats/parse.go b/internal/stats/parse.go index 1041182..a36b41c 100644 --- a/internal/stats/parse.go +++ b/internal/stats/parse.go @@ -11,18 +11,36 @@ import ( type Event struct { Time time.Time Client netip.Addr + Port string // the client's port: with Client and Name, it ties an RPZ rewrite to its response View string // "" without views Name string // lowercase, without the final dot ("." for the root) Type string // e.g. A, AAAA, PTR RCode string // e.g. NOERROR, NXDOMAIN } +// Rewrite is a response policy (RPZ) rewrite named logged: blocking. +type Rewrite struct { + Time time.Time + Client netip.Addr + Port string + View string + Name string // the query name, normalized like Event.Name + Action string // NXDOMAIN, NODATA, PASSTHRU, DROP, ... + Via string // the policy zone entry: OWNER.ZONE +} + // responseRe matches named's response log lines (BIND 9.20, print-time): // // 07-Oct-2026 19:46:22.195 client @0x7135... 10.10.0.37#51709 (nas.adbraun.lol): view internal: response: nas.adbraun.lol IN A NOERROR 1 0 1 +E(0)K (10.10.0.2) // // The "view NAME: " part is missing when no views are configured. -var responseRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#\d+ \(.*?\): (?:view (.+?): )?response: (\S+) (\S+) (\S+) ([A-Z0-9]+) `) +var responseRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#(\d+) \(.*?\): (?:view (.+?): )?response: (\S+) (\S+) (\S+) ([A-Z0-9]+) `) + +// rewriteRe matches named's RPZ rewrite lines (category rpz), logged just +// before the response line of the same query: +// +// 09-Oct-2026 23:49:13.698 client @0x7135... 10.10.0.37#54468 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid +var rewriteRe = regexp.MustCompile(`^(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3}) client @0x[0-9a-fA-F]+ (\S+)#(\d+) \((.*?)\): (?:view (.+?): )?rpz \S+ (\S+) rewrite \S+ via (\S+)`) const logTimeLayout = "02-Jan-2006 15:04:05.000" @@ -44,13 +62,31 @@ func ParseResponse(line string, loc *time.Location) (ev Event, ok bool) { return Event{ Time: t, Client: client.Unmap(), - View: m[3], - Name: normalizeName(m[4]), - Type: m[6], - RCode: m[7], + Port: m[3], + View: m[4], + Name: normalizeName(m[5]), + Type: m[7], + RCode: m[8], }, true } +// ParseRewrite parses an RPZ rewrite log line. +func ParseRewrite(line string, loc *time.Location) (rw Rewrite, ok bool) { + m := rewriteRe.FindStringSubmatch(line) + if m == nil { + return rw, false + } + t, err := time.ParseInLocation(logTimeLayout, m[1], loc) + if err != nil { + return rw, false + } + client, err := netip.ParseAddr(m[2]) + if err != nil { + return rw, false + } + return Rewrite{Time: t, Client: client.Unmap(), Port: m[3], Name: normalizeName(m[4]), View: m[5], Action: m[6], Via: strings.ToLower(m[7])}, true +} + func normalizeName(n string) string { n = strings.ToLower(n) if n == "." { diff --git a/internal/stats/query.go b/internal/stats/query.go index 1b2bf1c..6153c75 100644 --- a/internal/stats/query.go +++ b/internal/stats/query.go @@ -153,14 +153,19 @@ type Summary struct { Queries int64 `json:"queries"` Per string `json:"per"` // minute, hour or day } `json:"peak,omitempty"` - RCodes []Entry `json:"rcodes"` - QTypes []Entry `json:"qtypes"` - Views []Entry `json:"views"` - Zones []Entry `json:"zones"` - Answers []Entry `json:"answers"` // authoritative, recursive - Transport []Entry `json:"transport"` // udp, tcp - CacheHitRatio *float64 `json:"cache_hit_ratio,omitempty"` + RCodes []Entry `json:"rcodes"` + QTypes []Entry `json:"qtypes"` + Views []Entry `json:"views"` + Zones []Entry `json:"zones"` + Answers []Entry `json:"answers"` // authoritative, recursive + Transport []Entry `json:"transport"` // udp, tcp + CacheHitRatio *float64 `json:"cache_hit_ratio,omitempty"` + // NXDomainRatio leaves out answers blocking gave (which are + // NXDOMAIN too); Blocked counts those (docs/blocking.md). NXDomainRatio *float64 `json:"nxdomain_ratio,omitempty"` + Blocked int64 `json:"blocked"` + BlockedRatio *float64 `json:"blocked_ratio,omitempty"` + Blocklists []Entry `json:"blocklists,omitempty"` Tops map[string][]Entry `json:"tops,omitempty"` } @@ -181,9 +186,14 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int r := float64(h) / float64(h+m) s.CacheHitRatio = &r } + s.Blocked, s.Blocklists = all["blocked"], sorted(all, "blocklist:") if s.Queries > 0 { - r := float64(all["rcode:NXDOMAIN"]) / float64(s.Queries) + r := float64(max(all["rcode:NXDOMAIN"]-s.Blocked, 0)) / float64(s.Queries) s.NXDomainRatio = &r + if s.Blocked > 0 { + b := float64(s.Blocked) / float64(s.Queries) + s.BlockedRatio = &b + } } per, err := q.Store.series(ctx, res, from, to, "queries") if err != nil { @@ -209,7 +219,7 @@ func (q *Querier) Summary(ctx context.Context, since time.Duration, topLimit int } if topLimit > 0 { s.Tops = map[string][]Entry{} - for _, k := range []string{KindName, KindClient, KindNXDomain} { + for _, k := range []string{KindName, KindClient, KindNXDomain, KindBlocked} { es, err := q.Top(ctx, k, since, topLimit) if err != nil { return nil, err @@ -246,7 +256,7 @@ func (q *Querier) Breakdown(ctx context.Context, group string, since time.Durati } // Groups are the counter groups Breakdown accepts. -var Groups = []string{"qtype", "rcode", "view", "zone", "answer", "transport", "rtt", "dnssec", "cache", "ratelimit"} +var Groups = []string{"qtype", "rcode", "view", "zone", "answer", "transport", "rtt", "dnssec", "cache", "ratelimit", "blocklist"} // Timeline is a set of series over time. type Timeline struct { diff --git a/internal/store/blocklists.go b/internal/store/blocklists.go new file mode 100644 index 0000000..24cc315 --- /dev/null +++ b/internal/store/blocklists.go @@ -0,0 +1,42 @@ +package store + +import ( + "context" + "database/sql" + + "jarl/internal/model" +) + +// BlocklistStatuses returns the download status of every blocklist that +// has one. +func (s *Store) BlocklistStatuses(ctx context.Context) (map[int64]*model.BlocklistStatus, error) { + rows, err := s.db.QueryContext(ctx, `SELECT id, url, etag, last_modified, checked_at, updated_at, entries, records, skipped, sha256, error FROM blocklist_status`) + if err != nil { + return nil, err + } + out := map[int64]*model.BlocklistStatus{} + for rows.Next() { + st := &model.BlocklistStatus{} + var checked, updated sql.NullString + if err := rows.Scan(&st.ID, &st.URL, &st.ETag, &st.LastModified, &checked, &updated, &st.Entries, &st.Records, &st.Skipped, &st.SHA256, &st.Error); err != nil { + rows.Close() + return nil, err + } + st.CheckedAt, st.UpdatedAt = parseTSPtr(checked), parseTSPtr(updated) + out[st.ID] = st + } + return out, closeRows(rows) +} + +// SetBlocklistStatus stores a blocklist's download status. +func (s *Store) SetBlocklistStatus(ctx context.Context, st *model.BlocklistStatus) error { + _, err := s.db.ExecContext(ctx, `INSERT OR REPLACE INTO blocklist_status (id, url, etag, last_modified, checked_at, updated_at, entries, records, skipped, sha256, error) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, + st.ID, st.URL, st.ETag, st.LastModified, tsPtr(st.CheckedAt), tsPtr(st.UpdatedAt), st.Entries, st.Records, st.Skipped, st.SHA256, st.Error) + return err +} + +// DeleteBlocklistStatus forgets a deleted blocklist's status. +func (s *Store) DeleteBlocklistStatus(ctx context.Context, id int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM blocklist_status WHERE id = ?`, id) + return err +} diff --git a/internal/store/state.go b/internal/store/state.go index 595c14b..8c628db 100644 --- a/internal/store/state.go +++ b/internal/store/state.go @@ -149,6 +149,44 @@ func loadState(ctx context.Context, e execer) (*model.State, error) { if err := closeRows(rows); err != nil { return nil, err } + + rows, err = e.QueryContext(ctx, `SELECT id, name, url, view_ids, disabled, comment, version, created_at, updated_at, created_by, updated_by FROM blocklists`) + if err != nil { + return nil, err + } + for rows.Next() { + b := &model.Blocklist{} + var views, ca, ua string + if err := rows.Scan(&b.ID, &b.Name, &b.URL, &views, &b.Disabled, &b.Comment, &b.Version, &ca, &ua, &b.CreatedBy, &b.UpdatedBy); err != nil { + rows.Close() + return nil, err + } + json.Unmarshal([]byte(views), &b.ViewIDs) + b.CreatedAt, b.UpdatedAt = parseTS(ca), parseTS(ua) + st.Blocklists[b.ID] = b + } + if err := closeRows(rows); err != nil { + return nil, err + } + + rows, err = e.QueryContext(ctx, `SELECT id, name, action, exact, view_ids, comment, version, created_at, updated_at, created_by, updated_by FROM block_rules`) + if err != nil { + return nil, err + } + for rows.Next() { + r := &model.BlockRule{} + var views, ca, ua string + if err := rows.Scan(&r.ID, &r.Name, &r.Action, &r.Exact, &views, &r.Comment, &r.Version, &ca, &ua, &r.CreatedBy, &r.UpdatedBy); err != nil { + rows.Close() + return nil, err + } + json.Unmarshal([]byte(views), &r.ViewIDs) + r.CreatedAt, r.UpdatedAt = parseTS(ca), parseTS(ua) + st.BlockRules[r.ID] = r + } + if err := closeRows(rows); err != nil { + return nil, err + } return st, nil } @@ -166,7 +204,7 @@ func (t *Tx) SaveState(old, new *model.State) error { // ReplaceState replaces the stored configuration with st (used by import // and tests). func (t *Tx) ReplaceState(st *model.State) error { - for _, tbl := range []string{"views", "acls", "tsig_keys", "zones", "records", "options"} { + for _, tbl := range []string{"views", "acls", "tsig_keys", "zones", "records", "options", "blocklists", "block_rules"} { if _, err := t.tx.ExecContext(t.ctx, "DELETE FROM "+tbl); err != nil { return err } @@ -226,11 +264,32 @@ func saveState(ctx context.Context, e execer, old, new *model.State) error { }, func(id int64) error { return del(ctx, e, "records", id) }); err != nil { return err } - return diffMap(old.Options, new.Options, func(o *model.Option) error { + if err := diffMap(old.Options, new.Options, func(o *model.Option) error { _, err := e.ExecContext(ctx, `INSERT OR REPLACE INTO options (id, scope, scope_id, name, value, position, comment, version, created_at, updated_at, created_by, updated_by) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`, o.ID, o.Scope, o.ScopeID, o.Name, o.Value, o.Position, o.Comment, o.Version, ts(o.CreatedAt), ts(o.UpdatedAt), o.CreatedBy, o.UpdatedBy) return err - }, func(id int64) error { return del(ctx, e, "options", id) }) + }, func(id int64) error { return del(ctx, e, "options", id) }); err != nil { + return err + } + if err := diffMap(old.Blocklists, new.Blocklists, func(b *model.Blocklist) error { + _, err := e.ExecContext(ctx, `INSERT OR REPLACE INTO blocklists (id, name, url, view_ids, disabled, comment, version, created_at, updated_at, created_by, updated_by) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, + b.ID, b.Name, b.URL, jsonText(nonNilIDs(b.ViewIDs)), b.Disabled, b.Comment, b.Version, ts(b.CreatedAt), ts(b.UpdatedAt), b.CreatedBy, b.UpdatedBy) + return err + }, func(id int64) error { return del(ctx, e, "blocklists", id) }); err != nil { + return err + } + return diffMap(old.BlockRules, new.BlockRules, func(r *model.BlockRule) error { + _, err := e.ExecContext(ctx, `INSERT OR REPLACE INTO block_rules (id, name, action, exact, view_ids, comment, version, created_at, updated_at, created_by, updated_by) VALUES (?,?,?,?,?,?,?,?,?,?,?)`, + r.ID, r.Name, r.Action, r.Exact, jsonText(nonNilIDs(r.ViewIDs)), r.Comment, r.Version, ts(r.CreatedAt), ts(r.UpdatedAt), r.CreatedBy, r.UpdatedBy) + return err + }, func(id int64) error { return del(ctx, e, "block_rules", id) }) +} + +func nonNilIDs(ids []int64) []int64 { + if ids == nil { + return []int64{} + } + return ids } func nonNil(s []string) []string { diff --git a/internal/store/store.go b/internal/store/store.go index 683054b..8690c4f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -263,6 +263,43 @@ CREATE TABLE recovery_codes ( // 5: disabled records, kept but left out of the zone. ` ALTER TABLE records ADD COLUMN disabled INTEGER NOT NULL DEFAULT 0; +`, + // 6: blocking (docs/blocking.md). blocklist_status holds what jarld + // learns downloading lists; it is not configuration. + ` +CREATE TABLE blocklists ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + url TEXT NOT NULL, + view_ids TEXT NOT NULL DEFAULT '[]', + disabled INTEGER NOT NULL DEFAULT 0, + comment TEXT NOT NULL DEFAULT '', + version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT '' +); +CREATE TABLE block_rules ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + action TEXT NOT NULL, + exact INTEGER NOT NULL DEFAULT 0, + view_ids TEXT NOT NULL DEFAULT '[]', + comment TEXT NOT NULL DEFAULT '', + version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT '' +); +CREATE TABLE blocklist_status ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL DEFAULT '', + etag TEXT NOT NULL DEFAULT '', + last_modified TEXT NOT NULL DEFAULT '', + checked_at TEXT, + updated_at TEXT, + entries INTEGER NOT NULL DEFAULT 0, + records INTEGER NOT NULL DEFAULT 0, + skipped INTEGER NOT NULL DEFAULT 0, + sha256 TEXT NOT NULL DEFAULT '', + error TEXT NOT NULL DEFAULT '' +); `, } diff --git a/internal/validate/blocking.go b/internal/validate/blocking.go new file mode 100644 index 0000000..ff950d1 --- /dev/null +++ b/internal/validate/blocking.go @@ -0,0 +1,93 @@ +package validate + +import ( + "strings" + + "jarl/internal/model" +) + +// checkBlocking checks blocklists and block/allow rules (docs/blocking.md). +func (c *checker) checkBlocking() { + viewNames := func(ids []int64) string { + var names []string + for _, id := range ids { + if v := c.s.Views[id]; v != nil { + names = append(names, v.Name) + } + } + return strings.Join(names, ", ") + } + used := map[int64]bool{} // views (0: the server) with blocking + // what describes where something applies, and warns when it can't + // have any effect. + effective := func(objType string, id int64, label string, ids []int64) []int64 { + for _, vid := range ids { + if c.s.Views[vid] == nil { + c.errf("blocking_view_missing", objType, id, "%s names view %d, which doesn't exist", label, vid) + } + } + views := c.s.BlockingViews(ids) + if len(views) == 0 { + c.warnf("blocking_no_view", objType, id, "%s has no effect: no view offers recursion, and blocking only changes answers to recursive queries", label) + } + var idle []int64 + for _, vid := range views { + if !c.s.RecursionOn(vid) { + idle = append(idle, vid) + } + } + if len(idle) > 0 { + c.warnf("blocking_no_recursion", objType, id, "%s has no effect in view %s: it doesn't offer recursion, and blocking only changes answers to recursive queries", label, viewNames(idle)) + } + for _, vid := range views { + used[vid] = true + } + return views + } + for _, b := range c.s.SortedBlocklists() { + if !b.Disabled { + effective("blocklist", b.ID, "blocklist "+b.Name, b.ViewIDs) + } + } + // One rule per name and view: two would compete for the same entries + // of the policy zone. + seen := map[string]*model.BlockRule{} // view|name -> first rule + for _, r := range c.s.SortedBlockRules() { + label := "the rule for " + r.Name + for _, vid := range effective("blockrule", r.ID, label, r.ViewIDs) { + key := c.blockingScope(vid) + "|" + r.Name + if first := seen[key]; first != nil && first.ID != r.ID { + where := "" + if v := c.s.Views[vid]; v != nil { + where = " in view " + v.Name + } + c.errf("blocking_rule_duplicate", "blockrule", r.ID, "there are two rules for %s%s (%s and %s); keep one", r.Name, where, first.Action, r.Action) + continue + } + seen[key] = r + } + } + // Jarl writes the response-policy statement where blocking is used; + // named allows one per view (or for the server). + for vid := range used { + if o := c.s.FindOption(model.ScopeView, vid, "response-policy"); vid != 0 && o != nil { + p := c.errf("blocking_response_policy", "option", o.ID, "view %s has a response-policy option, but Jarl's blocking writes that statement there; remove the option (or the blocking that applies to this view)", c.s.Views[vid].Name) + p.View = c.s.Views[vid].Name + } + } + if o := c.s.FindOption(model.ScopeOptions, 0, "response-policy"); o != nil && len(used) > 0 { + c.errf("blocking_response_policy", "option", o.ID, "the server has a response-policy option, which Jarl's blocking replaces wherever it is used; remove the option, or set up your policy zones with Jarl's blocking instead") + } + for _, z := range c.s.Zones { + if n := strings.TrimSuffix(z.Name, "."); n == "jarl.invalid" || strings.HasSuffix(n, ".jarl.invalid") { + c.zoneProblem(z, Error, "zone_name_reserved", "zone names under jarl.invalid are reserved for Jarl's blocking") + } + } +} + +func (c *checker) blockingScope(viewID int64) string { + if v := c.s.Views[viewID]; v != nil { + return v.Name + } + return "" +} diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 05d8dd4..2c5c945 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -137,6 +137,7 @@ func State(s *model.State, opts Options) []Problem { c.checkOptions() c.checkSecurity() c.checkDNSSEC() + c.checkBlocking() sortProblems(c.problems) return c.problems } diff --git a/internal/web/static/assets/app.js b/internal/web/static/assets/app.js index 689f863..07b721c 100644 --- a/internal/web/static/assets/app.js +++ b/internal/web/static/assets/app.js @@ -17,6 +17,7 @@ import * as users from "./pages/users.js"; import * as account from "./pages/account.js"; import * as search from "./pages/search.js"; import * as stats from "./pages/stats.js"; +import * as blocking from "./pages/blocking.js"; const routes = [ { pattern: /^$/, page: dashboard, nav: "dashboard" }, @@ -34,12 +35,14 @@ const routes = [ { pattern: /^account$/, page: account, nav: "account" }, { pattern: /^search$/, page: search, nav: "" }, { pattern: /^stats$/, page: stats, nav: "stats" }, + { pattern: /^blocking$/, page: blocking, nav: "blocking" }, ]; const navItems = [ ["dashboard", "#/", "Dashboard"], ["stats", "#/stats", "Statistics"], ["zones", "#/zones", "Zones"], + ["blocking", "#/blocking", "Blocking"], ["views", "#/views", "Views"], ["acls", "#/acls", "ACLs"], ["keys", "#/keys", "TSIG keys"], diff --git a/internal/web/static/assets/pages/blocking.js b/internal/web/static/assets/pages/blocking.js new file mode 100644 index 0000000..4350ff3 --- /dev/null +++ b/internal/web/static/assets/pages/blocking.js @@ -0,0 +1,208 @@ +// Blocking: your block/allow rules and blocklists (docs/blocking.md). +import { h, mount, table, input, select, checkbox, toast, errorBox, badge, confirmDialog, relTime, time } from "../ui.js"; +import { get, change, describeResult, draftQuery, can, api } from "../api.js"; +import { rerender } from "../app.js"; +import { pendingBadge } from "./zones.js"; + +// Well-known lists in Adblock format (they say which entries cover +// subdomains). Sizes change; these are rough guides. +const suggestions = [ + ["hagezi-light", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/light.txt", "Hagezi Light: small, very few false positives"], + ["hagezi-multi", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/multi.txt", "Hagezi Multi Normal: ads, tracking, malware; a good default"], + ["hagezi-pro", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt", "Hagezi Pro: more thorough, occasionally blocks something wanted"], + ["oisd-small", "https://small.oisd.nl/", "OISD Small: ads and tracking, very few false positives"], +]; + +const stateBadge = { + ok: ["in use", "ok"], waiting: ["downloading", "accent"], error: ["update failed", "err"], + disabled: ["disabled", ""], idle: ["no effect", "warn"], +}; + +let refreshTimer = null; + +export async function render(el, params, query) { + clearTimeout(refreshTimer); + const [lists, rules, views] = await Promise.all([get("/blocking/lists", draftQuery()), get("/blocking/rules", draftQuery()), get("/views")]); + const writable = can("blocking", true); + const recursive = lists.concat(rules).find((x) => x.auto_views); + const autoViews = recursive ? recursive.views : null; + + const checkOut = h("div"); + const checkName = input({ type: "search", placeholder: "ads.example.com", class: "mono", value: (query && query.get("check")) || "" }); + const runCheck = async (ev) => { + if (ev) ev.preventDefault(); + const name = checkName.value.trim(); + if (!name) return; + try { + const res = await get("/blocking/check", { name }); + mount(checkOut, checkResult(name, res, writable)); + } catch (e) { mount(checkOut, errorBox(e)); } + }; + + mount(el, + h("div", { class: "page-head" }, + h("div", { class: "grow" }, h("h1", null, "Blocking"), + h("div", { class: "muted" }, "Answer \"doesn't exist\" for ads, trackers and malware, for the devices that use this server to resolve names. Your rules come first, then the blocklists."))), + h("div", { class: "panel" }, + h("form", { class: "actions", onsubmit: runCheck }, + h("label", { class: "muted" }, "Check a name"), checkName, h("button", { type: "submit" }, "Check")), + checkOut), + rulesPanel(rules, views, autoViews, writable), + listsPanel(lists, views, autoViews, writable)); + if (checkName.value) runCheck(); + + // Lists are downloaded in the background: show when they're done. + if (lists.some((l) => l.state === "waiting")) { + refreshTimer = setTimeout(() => { if (location.hash.startsWith("#/blocking") && el.isConnected) rerender(); }, 4000); + } +} + +function checkResult(name, res, writable) { + if (!res.length) return h("p", { class: "muted" }, "Blocking isn't used in any view yet: add a rule or a blocklist below."); + const blocked = res.some((m) => m.result === "blocked"); + return h("div", null, + h("ul", { class: "plain" }, res.map((m) => h("li", null, + m.view ? h("span", { class: "muted" }, "view " + m.view + ": ") : null, + m.result === "blocked" ? badge("blocked", "err") : m.result === "allowed" ? badge("allowed", "ok") : badge("not blocked", ""), + " ", + m.list ? ["by blocklist ", h("strong", null, m.list), " (entry ", h("code", null, m.entry), ")"] + : m.result !== "none" ? ["by your rule for ", h("code", null, ruleName(m.entry)), m.entry.startsWith("*.") ? " (names below it)" : ""] : null))), + blocked && writable ? h("p", null, h("button", { class: "small", onclick: () => addRule(name, "allow", false, []) }, "Allow " + name), + h("span", { class: "muted small" }, " adds an allow rule for it and the names below it, which beats every list.")) : null); +} + +const ruleName = (entry) => entry.replace(/^\*\./, ""); + +async function addRule(name, action, exact, views, comment) { + try { + const r = await change("POST", "/blocking/rules", { name, action, exact, views, comment: comment || "" }); + toast(describeResult(r)); + rerender(); + return true; + } catch (e) { toast(e.message, "err", 8000); return false; } +} + +// viewPicker returns checkboxes for the views (none ticked: automatic). +function viewPicker(views, autoViews, selected) { + if (!views.length) return { el: null, value: () => [] }; + const boxes = views.map((v) => checkbox(v.name, (selected || []).includes(v.name))); + const hint = autoViews && autoViews.length ? "none ticked: " + autoViews.join(", ") + " (every view that offers recursion)" : "none ticked: every view that offers recursion"; + return { + el: h("div", { class: "f" }, h("label", null, "Views"), h("div", { class: "actions" }, boxes, h("span", { class: "muted small" }, hint))), + value: () => views.filter((v, i) => boxes[i].input.checked).map((v) => v.name), + }; +} + +function viewsCell(x) { + if (!x.views.length) return h("span", { class: "muted" }, "none"); + return x.auto_views ? [x.views.join(", "), h("span", { class: "muted small" }, " (auto)")] : x.views.join(", "); +} + +function rulesPanel(rules, views, autoViews, writable) { + const name = input({ placeholder: "ads.example.com", class: "mono" }); + const action = select([["block", "Block"], ["allow", "Allow"]], "block"); + const exact = checkbox("only this exact name", false); + const comment = input({ placeholder: "optional" }); + const vp = viewPicker(views, autoViews); + const err = h("div"); + const form = h("form", { class: "inline-form", onsubmit: async (ev) => { + ev.preventDefault(); + try { + const r = await change("POST", "/blocking/rules", { name: name.value.trim(), action: action.value, exact: exact.input.checked, views: vp.value(), comment: comment.value }); + toast(describeResult(r)); + rerender(); + } catch (e) { mount(err, errorBox(e)); } + } }, + h("div", { class: "f" }, h("label", null, "Action"), action), + h("div", { class: "f grow" }, h("label", null, "Domain"), name), + h("div", { class: "f" }, h("label", null, " "), exact), + h("div", { class: "f" }, h("label", null, "Comment"), comment), + vp.el, + h("div", { class: "f" }, h("label", null, " "), h("button", { class: "primary", type: "submit" }, "Add rule"))); + return h("div", { class: "panel" }, + h("h2", null, "Your rules"), + h("p", { class: "muted small" }, "A rule covers the domain and every name below it, unless it's for the exact name only. Rules beat every blocklist, and the most specific rule wins: allowing cdn.example.com overrides blocking example.com for cdn.example.com and the names below it."), + writable ? [form, err] : null, + table([ + { label: "Action", render: (r) => [r.action === "block" ? badge("block", "err") : badge("allow", "ok"), r.pending ? [" ", pendingBadge(r.pending)] : null] }, + { label: "Domain", render: (r) => [h("span", { class: "mono" }, r.name), r.exact ? null : h("span", { class: "muted small" }, " and below")] }, + views.length ? { label: "Views", render: viewsCell } : null, + { label: "Comment", render: (r) => h("span", { class: "muted" }, r.comment || "") }, + writable ? { label: "", render: (r) => r.pending === "delete" ? "" : h("button", { class: "small danger", onclick: () => del("/blocking/rules/" + r.id, "the rule for " + r.name) }, "Delete"), tdClass: "actions-cell" } : null, + ].filter(Boolean), rules, { empty: "No rules yet.", rowClass: (r) => r.pending ? "pending-" + r.pending : "" })); +} + +function listsPanel(lists, views, autoViews, writable) { + const name = input({ placeholder: "hagezi-multi", class: "mono", style: { width: "11em" } }); + const url = input({ placeholder: "https://…", class: "mono" }); + const vp = viewPicker(views, autoViews); + const err = h("div"); + const form = h("form", { class: "inline-form", onsubmit: async (ev) => { + ev.preventDefault(); + try { + const r = await change("POST", "/blocking/lists", { name: name.value.trim(), url: url.value.trim(), views: vp.value() }); + toast(describeResult(r) + (r.applied ? " — jarld downloads it now" : "")); + rerender(); + } catch (e) { mount(err, errorBox(e)); } + } }, + h("div", { class: "f" }, h("label", null, "Name"), name), + h("div", { class: "f grow" }, h("label", null, "URL"), url), + vp.el, + h("div", { class: "f" }, h("label", null, " "), h("button", { class: "primary", type: "submit" }, "Add list"))); + const used = new Set(lists.map((l) => l.url)); + const picks = suggestions.filter(([, u]) => !used.has(u)); + const totalMB = lists.reduce((a, l) => a + (l.memory_mb || 0), 0); + return h("div", { class: "panel" }, + h("h2", null, "Blocklists"), + h("p", { class: "muted small" }, "jarld downloads each list, checks it for changes daily, and keeps using the previous version if an update fails. Hosts, plain domain and Adblock (||domain^) lists work; Adblock entries also cover the names below them."), + writable ? [form, err] : null, + writable && picks.length ? h("p", { class: "small" }, h("span", { class: "muted" }, "Well-known lists: "), + picks.map(([n, u, d], i) => [i ? " · " : "", h("a", { href: "#", title: d + "\n" + u, onclick: (ev) => { ev.preventDefault(); name.value = n; url.value = u; name.focus(); } }, n)])) : null, + table([ + { label: "Name", render: (l) => [h("strong", { class: "mono" }, l.name), l.pending ? [" ", pendingBadge(l.pending)] : null] }, + { label: "State", render: (l) => { const [t, k] = stateBadge[l.state] || [l.state, ""]; return [badge(t, k), l.status && l.status.error ? h("div", { class: "small err-text" }, l.status.error) : null]; } }, + { label: "Domains", render: (l) => l.status && l.status.updated_at ? l.status.entries.toLocaleString() : "–", tdClass: "right" }, + views.length ? { label: "Views", render: viewsCell } : null, + { label: "Updated", render: (l) => l.status && l.status.updated_at ? h("span", { title: "checked " + time(l.status.checked_at) }, relTime(l.status.updated_at)) : "–", tdClass: "small" }, + { label: "Memory", render: (l) => l.memory_mb ? h("span", { title: "named's memory for this list, estimated (all its views)" }, "~" + l.memory_mb + " MB") : "–", tdClass: "small right" }, + { label: "URL", render: (l) => h("a", { href: l.url, target: "_blank", rel: "noopener", class: "small mono", title: l.url }, shortURL(l.url)) }, + writable ? { label: "", render: (l) => l.pending === "delete" ? "" : h("span", null, + h("button", { class: "small", disabled: l.state === "disabled" || !!l.pending, title: "Download now", onclick: () => refresh(l) }, "Update"), " ", + h("button", { class: "small", onclick: () => toggle(l) }, l.disabled ? "Enable" : "Disable"), " ", + h("button", { class: "small danger", onclick: () => del("/blocking/lists/" + l.id, "blocklist " + l.name) }, "Delete")), tdClass: "actions-cell" } : null, + ].filter(Boolean), lists, { empty: "No blocklists yet.", rowClass: (l) => l.pending ? "pending-" + l.pending : l.disabled ? "disabled" : "" }), + totalMB ? h("p", { class: "muted small" }, "named needs about " + totalMB + " MB of memory for these lists (an estimate). A list costs memory in every view it applies to, so pick views if some don't need it.") : null); +} + +function shortURL(u) { + const s = u.replace(/^https?:\/\//, ""); + return s.length > 48 ? s.slice(0, 22) + "…" + s.slice(-24) : s; +} + +async function refresh(l) { + toast("Downloading " + l.name + "…", "ok", 2000); + try { + const r = await api("POST", "/blocking/lists/" + l.id + "/refresh"); + if (r.status && r.status.error) toast(l.name + ": " + r.status.error, "err", 10000); + else toast(l.name + ": " + r.status.entries.toLocaleString() + " domains"); + rerender(); + } catch (e) { toast(e.message, "err", 8000); } +} + +async function toggle(l) { + try { + const r = await change("PATCH", "/blocking/lists/" + l.id, { disabled: !l.disabled }); + toast(describeResult(r)); + rerender(); + } catch (e) { toast(e.message, "err", 8000); } +} + +async function del(path, what) { + if (!(await confirmDialog("Delete", "Delete " + what + "?", "Delete", "danger"))) return; + try { + const r = await change("DELETE", path); + toast(describeResult(r)); + rerender(); + } catch (e) { toast(e.message, "err", 8000); } +} + diff --git a/internal/web/static/assets/pages/dashboard.js b/internal/web/static/assets/pages/dashboard.js index fea1c22..15b2975 100644 --- a/internal/web/static/assets/pages/dashboard.js +++ b/internal/web/static/assets/pages/dashboard.js @@ -40,6 +40,12 @@ export async function render(el) { h("strong", null, "The last backup failed "), time(b.last_error_at), ": " + b.last_error, h("div", { class: "small" }, "See 'journalctl -u jarl-backup' on the server."))); } + for (const e of st.blocklist_errors || []) { + parts.push(h("div", { class: "msg warn" }, + h("strong", null, "Blocklist " + e.name + " couldn't be updated: "), e.error + ". ", + e.updated_at ? ["named keeps using the version from ", time(e.updated_at), ". "] : "It isn't in use yet. ", + "jarld tries again within the hour. ", h("a", { href: "#/blocking" }, "Blocking"))); + } const tf = st.totp; if (tf && tf.missing && tf.missing.length) { parts.push(h("div", { class: "msg warn" }, diff --git a/internal/web/static/assets/pages/stats.js b/internal/web/static/assets/pages/stats.js index 1cc84b4..3adcca7 100644 --- a/internal/web/static/assets/pages/stats.js +++ b/internal/web/static/assets/pages/stats.js @@ -56,7 +56,8 @@ export async function render(el, params, query) { tile("Per second", s.avg_qps < 10 ? s.avg_qps.toFixed(2) : Math.round(s.avg_qps).toLocaleString(), "on average"), s.peak ? tile("Busiest " + s.peak.per, compact(s.peak.queries), fmtTime(s.peak.time)) : null, s.nxdomain_ratio != null ? tile("NXDOMAIN", pct(s.nxdomain_ratio), "of queries named no existing name") : null, - s.cache_hit_ratio != null ? tile("Cache hits", pct(s.cache_hit_ratio), "of recursive queries answered from cache") : null)); + s.cache_hit_ratio != null ? tile("Cache hits", pct(s.cache_hit_ratio), "of recursive queries answered from cache") : null, + s.blocked_ratio != null ? tile("Blocked", pct(s.blocked_ratio), compact(s.blocked) + " queries, by your rules and blocklists") : null)); // Volume over time: one series, so no legend box; the title names it. const q = timeline.series.find((x) => x.key === "queries") || { values: timeline.times.map(() => 0) }; @@ -84,7 +85,8 @@ export async function render(el, params, query) { listCard("Busiest clients", "", barList(t.client)), listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + "." })), listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + "." })), - t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null)); + t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null, + t.blocked.length ? listCard("Most blocked names", "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: (k) => "#/blocking?check=" + encodeURIComponent(k) })) : null)); } else { parts.push(h("div", { class: "msg info" }, "Top names and clients need the response log: set stats.query_log: true in /etc/jarl/jarl.yaml and restart jarl.")); } @@ -98,7 +100,8 @@ export async function render(el, params, query) { b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel })) : null, b.zone.length ? listCard("Zones", "queries per local zone; other = names in no local zone (recursion)", barList(b.zone, { label: (k) => (k === "-" ? "other" : k), href: (k) => (k === "-" ? null : "#/search?q=" + encodeURIComponent(k)) })) : null, b.answer.length ? listCard("Answers", "authoritative (your zones) or recursive (from the cache or the internet)", barList(b.answer)) : null, - rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null)); + rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null, + b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: () => "#/blocking" })) : null)); el.classList.remove("refreshing"); mount(el, parts); scheduleRefresh(el, params, query, gen); @@ -111,10 +114,10 @@ async function loadAll(since, heatSince, queryLog) { errors: get("/stats/timeline", { since, key: failures.map((f) => f.key).join(",") }), heat: get("/stats/heatmap", { since: heatSince }), }; - const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt"]; + const groups = ["qtype", "rcode", "view", "zone", "answer", "rtt", "blocklist"]; groups.forEach((g) => { p["b_" + g] = get("/stats/breakdown", { group: g, since }); }); if (queryLog) { - for (const k of ["name", "client", "nxdomain", "refused", "servfail"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 }); + for (const k of ["name", "client", "nxdomain", "refused", "servfail", "blocked"]) p["t_" + k] = get("/stats/top", { kind: k, since, limit: 15 }); } const keys = Object.keys(p); const vals = await Promise.all(Object.values(p)); diff --git a/internal/web/static/assets/style.css b/internal/web/static/assets/style.css index 08b2203..6a70624 100644 --- a/internal/web/static/assets/style.css +++ b/internal/web/static/assets/style.css @@ -168,7 +168,7 @@ textarea { width: 100%; min-height: 80px; resize: vertical; } .form-grid input, .form-grid select { width: 100%; max-width: 560px; } .inline-form { display: flex; gap: 6px; flex-wrap: wrap; align-items: flex-end; } .inline-form .f { display: flex; flex-direction: column; gap: 2px; } -.inline-form .f label { font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.3px; } +.inline-form .f label:not(.check) { font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.3px; } .inline-form .grow { flex: 1; min-width: 220px; } .inline-form .grow input { width: 100%; } .hint { color: var(--muted); font-size: 12px; margin-top: 4px; } @@ -442,4 +442,6 @@ footer.site a:hover { color: var(--accent); text-decoration-color: currentColor; .query-log code { background: var(--code-bg); padding: 1px 6px; border-radius: 4px; overflow-wrap: anywhere; } .query-log li.pending code { opacity: 0.85; } .err-text { color: var(--err); } +ul.plain { list-style: none; margin: 8px 0; padding: 0; } +ul.plain li { padding: 3px 0; } .query-details { margin: 8px 0; } diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 67f27b9..a2ab4c0 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -13,7 +13,7 @@ set -u ROOT=$(cd "$(dirname "$0")/.." && pwd) WORK=${WORK:-$(mktemp -d /tmp/jarl-e2e.XXXXXX)} E=$WORK -DNS=${DNS_PORT:-25353}; CTL=${CTL_PORT:-25953}; HTTP=${HTTP_PORT:-28053} +DNS=${DNS_PORT:-25353}; CTL=${CTL_PORT:-25953}; HTTP=${HTTP_PORT:-28053}; LISTS=${LISTS_PORT:-28054} if [ -n "${JARL_BIND_ROOT:-}" ]; then B=$JARL_BIND_ROOT export LD_LIBRARY_PATH=$(echo $B/usr/lib/*-linux-gnu | cut -d' ' -f1) @@ -30,8 +30,9 @@ done mkdir -p $E/legacy $E/cache $E/data JARLD=$E/jarld J=$E/jarl (cd "$ROOT" && go build -o $JARLD ./cmd/jarld && go build -o $J ./cmd/jarl) || exit 1 -NAMED_PID= JARLD_PID= +NAMED_PID= JARLD_PID= LISTS_PID= cleanup() { + [ -n "$LISTS_PID" ] && kill $LISTS_PID 2>/dev/null if [ "${KEEP:-}" = 1 ]; then echo "left running in $WORK (named $NAMED_PID, jarld $JARLD_PID)"; return; fi [ -n "$JARLD_PID" ] && kill $JARLD_PID 2>/dev/null [ -n "$NAMED_PID" ] && kill $NAMED_PID 2>/dev/null @@ -409,6 +410,60 @@ run "wipe the child" 0 $J zone dnssec wipe sub.adbraun.test --view external --ye run "delete the child" 0 $J zone delete sub.adbraun.test --view external --yes run "remove the delegation" 0 $J record delete adbraun.test sub NS --view external --yes +# Blocking (docs/blocking.md). It only changes answers to recursive +# queries, so these ask with recursion and look at which policy zone (if +# any) the answer came from. +rpz() { + $DIG +tries=1 +time=3 -p $DNS @127.0.0.1 "$1" A | awk '$4 == "SOA" && $1 ~ /jarl\.invalid\.$/ {print $1; f = 1} END {if (!f) print "none"}' +} +expect_rpz() { # DESC NAME WANT: named reloads in the background, so wait a little + local got + for i in $(seq 30); do got=$(rpz "$2"); [ "$got" = "$3" ] && break; sleep 0.2; done + if [ "$got" = "$3" ]; then ok "$1: $2 -> $3"; else bad "$1: $2 -> $got, want $3"; fi +} +mkdir -p $E/lists +printf '# test list\n||tracker.adbraun.test^\n0.0.0.0 ads.adbraun.test\nnot a domain\n' > $E/lists/list.txt +(cd $E/lists && exec python3 -m http.server --bind 127.0.0.1 $LISTS) > /dev/null 2>&1 & +LISTS_PID=$! +run "block add" 0 $J block add blocked.adbraun.test +expect_rpz "a rule blocks the name" blocked.adbraun.test rules.jarl.invalid. +expect_rpz "and the names below it" a.blocked.adbraun.test rules.jarl.invalid. +expect_rpz "other names are answered" www.adbraun.test none +run "named.conf has the policy" 0 grep -q 'response-policy' $E/etc-jarl/named.conf +for i in $(seq 25); do curl -s -o /dev/null http://127.0.0.1:$LISTS/list.txt && break; sleep 0.2; done +run "blocklist add" 0 $J blocklist add test http://127.0.0.1:$LISTS/list.txt +run "blocklist refresh" 0 $J blocklist refresh test +contains "test: 2 domains (1 lines skipped)" +expect_rpz "an Adblock entry blocks the names below it" x.tracker.adbraun.test test.blocklist.jarl.invalid. +expect_rpz "a hosts entry blocks the name" ads.adbraun.test test.blocklist.jarl.invalid. +expect_rpz "but not the names below it" sub.ads.adbraun.test none +run "blocklist list" 0 $J blocklist list +contains "^test *ok *2 " +run "block allow" 0 $J block allow ok.tracker.adbraun.test +expect_rpz "an allow rule beats the list" deep.ok.tracker.adbraun.test none +expect_rpz "the rest of the list still blocks" x.tracker.adbraun.test test.blocklist.jarl.invalid. +run "block check" 0 $J block check deep.ok.tracker.adbraun.test +contains "allowed by your rule" +# The test server dates the file to the second, and the download above +# remembers that date: change it a second later, as a real list would. +sleep 1.1 +printf '||new.adbraun.test^\n' > $E/lists/list.txt +run "blocklist refresh after the list changed" 0 $J blocklist refresh test +expect_rpz "the new version applies" x.new.adbraun.test test.blocklist.jarl.invalid. +expect_rpz "and replaces the old one" x.tracker.adbraun.test none +printf 'gone\n' > $E/lists/list.txt +run "a broken download fails (exit 2)" 2 $J blocklist refresh test --force +contains "no domains found" +expect_rpz "and the previous version stays" x.new.adbraun.test test.blocklist.jarl.invalid. +run "blocklist disable" 0 $J blocklist disable test +expect_rpz "a disabled list doesn't block" x.new.adbraun.test none +run "blocklist delete" 0 $J blocklist delete test --yes +run "block delete" 0 $J block delete blocked.adbraun.test +expect_rpz "a deleted rule doesn't block" blocked.adbraun.test none +run "block delete (allow rule)" 0 $J block delete ok.tracker.adbraun.test +run "no blocking left in named.conf" 1 grep -q 'response-policy' $E/etc-jarl/named.conf +kill $LISTS_PID 2>/dev/null; LISTS_PID= + run "status at the end" 0 $J status contains "Drift:.*none"