package engine import ( "context" "errors" "fmt" "io" "io/fs" "net/http" "os" "path/filepath" "slices" "strconv" "strings" "time" "jarl/internal/blocking" "jarl/internal/model" "jarl/internal/render" "jarl/internal/version" ) // BlocklistLimits bound blocklist downloads (blocking in jarl.yaml). type BlocklistLimits struct { Refresh time.Duration // how often a list is checked for changes RetryAfter time.Duration // after a failed attempt MaxBytes int64 // per download MaxEntries int // domains per list Timeout time.Duration // per download } // DefaultBlocklistLimits are the limits without configuration. func DefaultBlocklistLimits() BlocklistLimits { return BlocklistLimits{Refresh: 24 * time.Hour, RetryAfter: time.Hour, MaxBytes: 64 << 20, MaxEntries: 1_000_000, Timeout: 2 * time.Minute} } // KickBlocklists asks the download loop to look for work now (a list was // added or changed). func (e *Engine) KickBlocklists() { select { case e.blKick <- struct{}{}: default: } } // BlocklistKicks delivers KickBlocklists' signals to the download loop. func (e *Engine) BlocklistKicks() <-chan struct{} { return e.blKick } // ensureBlocklistFiles creates missing blocklist zone files as empty // policy zones, so named can load its configuration before a list has // been downloaded. Existing files are never touched. func (e *Engine) ensureBlocklistFiles(paths []string) error { var errs []error for _, path := range paths { if _, err := os.Stat(path); err == nil { continue } if err := e.deploy.writeNew(path, []byte(blocking.Placeholder()), 0o640); err != nil && !errors.Is(err, fs.ErrExist) { errs = append(errs, err) } } return errors.Join(errs...) } // RefreshDue downloads the blocklists that are due: never tried, URL // changed, file missing or empty (the cache was cleared), last check older // than Refresh, or a failed attempt older than RetryAfter. Lists that are // disabled or apply to no view are skipped. It also removes the files and // status of deleted lists. func (e *Engine) RefreshDue(ctx context.Context) { st, err := e.store.LoadState(ctx) if err != nil { e.log.Warn("blocklists: loading the configuration failed", "err", err) return } statuses, err := e.store.BlocklistStatuses(ctx) if err != nil { e.log.Warn("blocklists: loading their status failed", "err", err) return } out := render.Render(st, e.cfg.Paths, e.cfg.Extras) if err := e.ensureBlocklistFiles(out.Blocklists); err != nil { e.log.Warn("blocklists: creating empty zone files failed", "err", err) } e.cleanBlocklists(ctx, st, statuses) now := e.Now() for _, b := range st.SortedBlocklists() { if b.Disabled || len(st.BlockingViews(b.ViewIDs)) == 0 { continue } s := statuses[b.ID] due := s == nil || s.URL != b.URL || s.CheckedAt == nil if !due { wait := e.Limits.Refresh if s.Error != "" { wait = e.Limits.RetryAfter } due = now.Sub(*s.CheckedAt) >= wait || s.Error == "" && isPlaceholder(e.cfg.Paths.BlocklistFile(b.ID)) } if !due { continue } if ctx.Err() != nil { return } if _, err := e.RefreshBlocklist(ctx, b.ID, false); err != nil && ctx.Err() == nil { e.log.Warn("blocklist update failed; named keeps using the previous version", "list", b.Name, "err", err) } } } // isPlaceholder reports whether a list's zone file is missing or still the // empty policy zone written before its download. func isPlaceholder(path string) bool { f, err := os.Open(path) if err != nil { return true } defer f.Close() head := make([]byte, len(blocking.Placeholder())) n, _ := io.ReadFull(f, head) return string(head[:n]) == blocking.Placeholder() } // cleanBlocklists removes the zone files and status of deleted lists. func (e *Engine) cleanBlocklists(ctx context.Context, st *model.State, statuses map[int64]*model.BlocklistStatus) { for id := range statuses { if st.Blocklists[id] == nil { e.store.DeleteBlocklistStatus(ctx, id) } } dir := e.cfg.Paths.BlocklistsDir() entries, err := os.ReadDir(dir) if err != nil { return } for _, ent := range entries { n := ent.Name() idText, ok := strings.CutPrefix(strings.TrimSuffix(n, ".rpz"), "list-") id, err := strconv.ParseInt(idText, 10, 64) if !ok || !strings.HasSuffix(n, ".rpz") || err != nil || st.Blocklists[id] != nil { continue } if err := os.Remove(filepath.Join(dir, n)); err == nil { e.log.Info("removed the zone file of a deleted blocklist", "file", n) } } } // errNotModified means the server said the list hasn't changed. var errNotModified = errors.New("not modified") // RefreshBlocklist downloads a list now and, when its content changed, // replaces its zone file and makes named reload it. force downloads even // when the server says nothing changed. A failure keeps the previous // version. It returns the new status. func (e *Engine) RefreshBlocklist(ctx context.Context, id int64, force bool) (*model.BlocklistStatus, error) { e.blMu.Lock() defer e.blMu.Unlock() st, err := e.store.LoadState(ctx) if err != nil { return nil, err } b := st.Blocklists[id] if b == nil { return nil, errf(KindNotFound, "blocklist %d does not exist", id) } statuses, err := e.store.BlocklistStatuses(ctx) if err != nil { return nil, err } prev := statuses[id] if prev == nil { prev = &model.BlocklistStatus{ID: id} } file := e.cfg.Paths.BlocklistFile(id) now := e.Now() next := *prev next.CheckedAt = &now conditional := !force && prev.URL == b.URL && prev.SHA256 != "" && !isPlaceholder(file) err = e.downloadBlocklist(ctx, st, b, &next, conditional, file) next.URL, next.Error = b.URL, "" if err != nil && !errors.Is(err, errNotModified) { next.Error = err.Error() } if serr := e.store.SetBlocklistStatus(ctx, &next); serr != nil && err == nil { err = serr } if errors.Is(err, errNotModified) { err = nil } return &next, err } // downloadBlocklist fetches, parses, checks and installs a list, filling // in s. It returns errNotModified when there is nothing to do. func (e *Engine) downloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist, s *model.BlocklistStatus, conditional bool, file string) error { ctx, cancel := context.WithTimeout(ctx, e.Limits.Timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.URL, nil) if err != nil { return err } req.Header.Set("User-Agent", "Jarl/"+version.Short()+" (blocklist download)") if conditional { if s.ETag != "" { req.Header.Set("If-None-Match", s.ETag) } if s.LastModified != "" { req.Header.Set("If-Modified-Since", s.LastModified) } } client := e.HTTP if client == nil { client = http.DefaultClient } resp, err := client.Do(req) if err != nil { return fmt.Errorf("download failed: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotModified && conditional { return errNotModified } if resp.StatusCode != http.StatusOK { return fmt.Errorf("download failed: the server answered %s", resp.Status) } zone := blocking.ListZone(b.Name) body := &limitedReader{r: resp.Body, left: e.Limits.MaxBytes} l, err := blocking.Parse(body, blocking.Limits{MaxEntries: e.Limits.MaxEntries, MaxNameLen: 253 - len(zone) - 3}) if body.over { return fmt.Errorf("the list is larger than %d MB (blocking.max_download_mb in jarl.yaml)", e.Limits.MaxBytes>>20) } if errors.Is(err, blocking.ErrTooManyEntries) { return fmt.Errorf("%v (blocking.max_entries in jarl.yaml)", err) } if err != nil { return fmt.Errorf("download failed: %w", err) } if len(l.Domains) == 0 { return fmt.Errorf("no domains found in %d lines: is this a blocklist in hosts, domains or Adblock format?", l.Lines) } entries, skipped := len(l.Domains), l.Skipped blocking.Compact(l) dir := filepath.Dir(file) if err := e.deploy.EnsureDir(dir, e.deploy.dirMode(dir)); err != nil { return err } tmp, err := os.CreateTemp(dir, ".list-*.rpz") if err != nil { return err } tmpName := tmp.Name() defer os.Remove(tmpName) // after a successful rename, there's nothing to remove records, hash, err := blocking.FeedZone(tmp, fmt.Sprintf("Blocklist %s from %s, %s", b.Name, b.URL, e.Now().UTC().Format(time.RFC3339)), uint32(e.Now().Unix()), l) if cerr := tmp.Close(); err == nil { err = cerr } if err != nil { return err } s.ETag, s.LastModified = resp.Header.Get("ETag"), resp.Header.Get("Last-Modified") s.Entries, s.Skipped = entries, skipped if hash == s.SHA256 && s.URL == b.URL && !isPlaceholder(file) { return errNotModified // same content: named keeps what it has } if e.tools != nil && !e.cfg.SkipCheck { res, err := e.tools.CheckZone(ctx, zone, tmpName) if err != nil { return fmt.Errorf("running named-checkzone failed: %w", err) } if !res.OK { msg := "named-checkzone rejected the list's zone" for _, m := range res.Messages { if m.Error { msg += ": " + m.Text break } } return errors.New(msg) } } if err := e.deploy.finish(tmpName, file, 0o640); err != nil { return err } now := e.Now() s.UpdatedAt, s.Records, s.SHA256 = &now, records, hash e.log.Info("blocklist updated", "list", b.Name, "domains", entries, "records", records, "skipped_lines", skipped) return e.reloadBlocklist(ctx, st, b) } // reloadBlocklist makes named load a list's new file in every view that // uses it. func (e *Engine) reloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist) error { if e.cfg.NoReload || e.tools == nil { return nil } if adopted, err := e.Adopted(); err != nil || !adopted { return nil // the file is ready for when named loads Jarl's configuration } for _, z := range render.PolicyZones(st, e.cfg.Paths) { if z.ListID != b.ID { continue } if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil { return fmt.Errorf("the list was downloaded, but named didn't reload it: %w", err) } } return nil } // limitedReader reads at most left bytes and notes when there was more. type limitedReader struct { r io.Reader left int64 over bool } func (l *limitedReader) Read(p []byte) (int, error) { if l.left <= 0 { var one [1]byte if n, _ := l.r.Read(one[:]); n > 0 { l.over = true } return 0, io.EOF } if int64(len(p)) > l.left { p = p[:l.left] } n, err := l.r.Read(p) l.left -= int64(n) return n, err } // CheckBlocking reports, for every view with blocking, whether name is // blocked or allowed there, and by which rule or list. It looks names up // the way named does: rules first, then the lists in order. func (e *Engine) CheckBlocking(st *model.State, name string) ([]blocking.Match, error) { n, err := blocking.NormalizeName(name) if err != nil { return nil, errf(KindBadRequest, "%v", err) } var out []blocking.Match feeds := map[string]string{} // file -> matching entry ("" none) var cur *blocking.Match for _, z := range render.PolicyZones(st, e.cfg.Paths) { if cur == nil || cur.View != z.View { out = append(out, blocking.Match{View: z.View, Result: "none"}) cur = &out[len(out)-1] } if cur.Result != "none" { continue // an earlier zone decided } if z.ListID == 0 { owner, act, ok := blocking.MatchRules(blocking.RuleEntries(render.ViewRules(st, z.ViewID)), n) if ok { cur.Entry, cur.RuleID = owner, ruleFor(st, z.ViewID, owner) cur.Result = map[string]string{blocking.ActionBlock: "blocked", blocking.ActionAllow: "allowed"}[act] } continue } entry, seen := feeds[z.File] if !seen { owner, ok, err := blocking.MatchFeed(z.File, n) if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } if ok { entry = owner } feeds[z.File] = entry } if entry != "" { cur.Result, cur.Entry, cur.List = "blocked", entry, st.Blocklists[z.ListID].Name } } return out, nil } // ruleFor finds the rule behind a rules-zone entry: the rule for the // entry's name, or (for entries added so the most specific rule wins) the // closest rule above it that covers the names below it. func ruleFor(st *model.State, viewID int64, owner string) int64 { name := strings.TrimPrefix(owner, "*.") wild := name != owner var best *model.BlockRule for _, r := range st.SortedBlockRules() { if !slices.Contains(st.BlockingViews(r.ViewIDs), viewID) { continue } if r.Name == name && (!wild || !r.Exact) { return r.ID } if !r.Exact && strings.HasSuffix(name, "."+r.Name) && (best == nil || len(r.Name) > len(best.Name)) { best = r } } if best != nil { return best.ID } return 0 }