jarl/internal/engine/blocklists.go

package engine

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"time"

	"jarl/internal/blocking"
	"jarl/internal/model"
	"jarl/internal/render"
	"jarl/internal/version"
)

// BlocklistLimits bound blocklist downloads (blocking in jarl.yaml).
type BlocklistLimits struct {
	Refresh    time.Duration // how often a list is checked for changes
	RetryAfter time.Duration // after a failed attempt
	MaxBytes   int64         // per download
	MaxEntries int           // domains per list
	Timeout    time.Duration // per download
}

// DefaultBlocklistLimits are the limits without configuration.
func DefaultBlocklistLimits() BlocklistLimits {
	return BlocklistLimits{Refresh: 24 * time.Hour, RetryAfter: time.Hour, MaxBytes: 64 << 20, MaxEntries: 1_000_000, Timeout: 2 * time.Minute}
}

// KickBlocklists asks the download loop to look for work now (a list was
// added or changed).
func (e *Engine) KickBlocklists() {
	select {
	case e.blKick <- struct{}{}:
	default:
	}
}

// BlocklistKicks delivers KickBlocklists' signals to the download loop.
func (e *Engine) BlocklistKicks() <-chan struct{} { return e.blKick }

// ensureBlocklistFiles creates missing blocklist zone files as empty
// policy zones, so named can load its configuration before a list has
// been downloaded. Existing files are never touched.
func (e *Engine) ensureBlocklistFiles(paths []string) error {
	var errs []error
	for _, path := range paths {
		if _, err := os.Stat(path); err == nil {
			continue
		}
		if err := e.deploy.writeNew(path, []byte(blocking.Placeholder()), 0o640); err != nil && !errors.Is(err, fs.ErrExist) {
			errs = append(errs, err)
		}
	}
	return errors.Join(errs...)
}

// RefreshDue downloads the blocklists that are due: never tried, URL
// changed, file missing or empty (the cache was cleared), last check older
// than Refresh, or a failed attempt older than RetryAfter. Lists that are
// disabled or apply to no view are skipped. It also removes the files and
// status of deleted lists.
func (e *Engine) RefreshDue(ctx context.Context) {
	st, err := e.store.LoadState(ctx)
	if err != nil {
		e.log.Warn("blocklists: loading the configuration failed", "err", err)
		return
	}
	statuses, err := e.store.BlocklistStatuses(ctx)
	if err != nil {
		e.log.Warn("blocklists: loading their status failed", "err", err)
		return
	}
	out := render.Render(st, e.cfg.Paths, e.extras(ctx))
	if err := e.ensureBlocklistFiles(out.Blocklists); err != nil {
		e.log.Warn("blocklists: creating empty zone files failed", "err", err)
	}
	e.cleanBlocklists(ctx, st, statuses)
	now := e.Now()
	for _, b := range st.SortedBlocklists() {
		if b.Disabled || len(st.BlockingViews(b.ViewIDs)) == 0 {
			continue
		}
		s := statuses[b.ID]
		due := s == nil || s.URL != b.URL || s.CheckedAt == nil
		if !due {
			wait := e.Limits.Refresh
			if s.Error != "" {
				wait = e.Limits.RetryAfter
			}
			due = now.Sub(*s.CheckedAt) >= wait || s.Error == "" && isPlaceholder(e.cfg.Paths.BlocklistFile(b.ID))
		}
		if !due {
			continue
		}
		if ctx.Err() != nil {
			return
		}
		if _, err := e.RefreshBlocklist(ctx, b.ID, false); err != nil && ctx.Err() == nil {
			e.log.Warn("blocklist update failed; named keeps using the previous version", "list", b.Name, "err", err)
		}
	}
}

// isPlaceholder reports whether a list's zone file is missing or still the
// empty policy zone written before its download.
func isPlaceholder(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		return true
	}
	defer f.Close()
	head := make([]byte, len(blocking.Placeholder()))
	n, _ := io.ReadFull(f, head)
	return string(head[:n]) == blocking.Placeholder()
}

// cleanBlocklists removes the zone files and status of deleted lists.
func (e *Engine) cleanBlocklists(ctx context.Context, st *model.State, statuses map[int64]*model.BlocklistStatus) {
	for id := range statuses {
		if st.Blocklists[id] == nil {
			e.store.DeleteBlocklistStatus(ctx, id)
		}
	}
	dir := e.cfg.Paths.BlocklistsDir()
	entries, err := os.ReadDir(dir)
	if err != nil {
		return
	}
	for _, ent := range entries {
		n := ent.Name()
		idText, ok := strings.CutPrefix(strings.TrimSuffix(n, ".rpz"), "list-")
		id, err := strconv.ParseInt(idText, 10, 64)
		if !ok || !strings.HasSuffix(n, ".rpz") || err != nil || st.Blocklists[id] != nil {
			continue
		}
		if err := os.Remove(filepath.Join(dir, n)); err == nil {
			e.log.Info("removed the zone file of a deleted blocklist", "file", n)
		}
	}
}

// errNotModified means the server said the list hasn't changed.
var errNotModified = errors.New("not modified")

// RefreshBlocklist downloads a list now and, when its content changed,
// replaces its zone file and makes named reload it. force downloads even
// when the server says nothing changed. A failure keeps the previous
// version. It returns the new status.
func (e *Engine) RefreshBlocklist(ctx context.Context, id int64, force bool) (*model.BlocklistStatus, error) {
	e.blMu.Lock()
	defer e.blMu.Unlock()
	st, err := e.store.LoadState(ctx)
	if err != nil {
		return nil, err
	}
	b := st.Blocklists[id]
	if b == nil {
		return nil, errf(KindNotFound, "blocklist %d does not exist", id)
	}
	statuses, err := e.store.BlocklistStatuses(ctx)
	if err != nil {
		return nil, err
	}
	prev := statuses[id]
	if prev == nil {
		prev = &model.BlocklistStatus{ID: id}
	}
	file := e.cfg.Paths.BlocklistFile(id)
	now := e.Now()
	next := *prev
	next.CheckedAt = &now
	conditional := !force && prev.URL == b.URL && prev.SHA256 != "" && !isPlaceholder(file)
	err = e.downloadBlocklist(ctx, st, b, &next, conditional, file)
	next.URL, next.Error = b.URL, ""
	if err != nil && !errors.Is(err, errNotModified) {
		next.Error = err.Error()
	}
	if serr := e.store.SetBlocklistStatus(ctx, &next); serr != nil && err == nil {
		err = serr
	}
	if errors.Is(err, errNotModified) {
		err = nil
	}
	return &next, err
}

// downloadBlocklist fetches, parses, checks and installs a list, filling
// in s. It returns errNotModified when there is nothing to do.
func (e *Engine) downloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist, s *model.BlocklistStatus, conditional bool, file string) error {
	ctx, cancel := context.WithTimeout(ctx, e.Limits.Timeout)
	defer cancel()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.URL, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Jarl/"+version.Short()+" (blocklist download)")
	if conditional {
		if s.ETag != "" {
			req.Header.Set("If-None-Match", s.ETag)
		}
		if s.LastModified != "" {
			req.Header.Set("If-Modified-Since", s.LastModified)
		}
	}
	client := e.HTTP
	if client == nil {
		client = http.DefaultClient
	}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("download failed: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified && conditional {
		return errNotModified
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("download failed: the server answered %s", resp.Status)
	}
	zone := blocking.ListZone(b.Name)
	body := &limitedReader{r: resp.Body, left: e.Limits.MaxBytes}
	l, err := blocking.Parse(body, blocking.Limits{MaxEntries: e.Limits.MaxEntries, MaxNameLen: 253 - len(zone) - 3})
	if body.over {
		return fmt.Errorf("the list is larger than %d MB (blocking.max_download_mb in jarl.yaml)", e.Limits.MaxBytes>>20)
	}
	if errors.Is(err, blocking.ErrTooManyEntries) {
		return fmt.Errorf("%v (blocking.max_entries in jarl.yaml)", err)
	}
	if err != nil {
		return fmt.Errorf("download failed: %w", err)
	}
	if len(l.Domains) == 0 {
		return fmt.Errorf("no domains found in %d lines: is this a blocklist in hosts, domains or Adblock format?", l.Lines)
	}
	entries, skipped := len(l.Domains), l.Skipped
	blocking.Compact(l)

	dir := filepath.Dir(file)
	if err := e.deploy.EnsureDir(dir, e.deploy.dirMode(dir)); err != nil {
		return err
	}
	tmp, err := os.CreateTemp(dir, ".list-*.rpz")
	if err != nil {
		return err
	}
	tmpName := tmp.Name()
	defer os.Remove(tmpName) // after a successful rename, there's nothing to remove
	records, hash, err := blocking.FeedZone(tmp, fmt.Sprintf("Blocklist %s from %s, %s", b.Name, b.URL, e.Now().UTC().Format(time.RFC3339)), uint32(e.Now().Unix()), l)
	if cerr := tmp.Close(); err == nil {
		err = cerr
	}
	if err != nil {
		return err
	}
	s.ETag, s.LastModified = resp.Header.Get("ETag"), resp.Header.Get("Last-Modified")
	s.Entries, s.Skipped = entries, skipped
	if hash == s.SHA256 && s.URL == b.URL && !isPlaceholder(file) {
		return errNotModified // same content: named keeps what it has
	}
	if e.tools != nil && !e.cfg.SkipCheck {
		res, err := e.tools.CheckZone(ctx, zone, tmpName)
		if err != nil {
			return fmt.Errorf("running named-checkzone failed: %w", err)
		}
		if !res.OK {
			msg := "named-checkzone rejected the list's zone"
			for _, m := range res.Messages {
				if m.Error {
					msg += ": " + m.Text
					break
				}
			}
			return errors.New(msg)
		}
	}
	if err := e.deploy.finish(tmpName, file, 0o640); err != nil {
		return err
	}
	now := e.Now()
	s.UpdatedAt, s.Records, s.SHA256 = &now, records, hash
	e.log.Info("blocklist updated", "list", b.Name, "domains", entries, "records", records, "skipped_lines", skipped)
	return e.reloadBlocklist(ctx, st, b)
}

// reloadBlocklist makes named load a list's new file in every view that
// uses it.
func (e *Engine) reloadBlocklist(ctx context.Context, st *model.State, b *model.Blocklist) error {
	if e.cfg.NoReload || e.tools == nil {
		return nil
	}
	if adopted, err := e.Adopted(); err != nil || !adopted {
		return nil // the file is ready for when named loads Jarl's configuration
	}
	for _, z := range render.PolicyZones(st, e.cfg.Paths) {
		if z.ListID != b.ID {
			continue
		}
		if err := e.tools.ReloadZone(ctx, z.Name, z.View); err != nil {
			return fmt.Errorf("the list was downloaded, but named didn't reload it: %w", err)
		}
	}
	return nil
}

// limitedReader reads at most left bytes and notes when there was more.
type limitedReader struct {
	r    io.Reader
	left int64
	over bool
}

func (l *limitedReader) Read(p []byte) (int, error) {
	if l.left <= 0 {
		var one [1]byte
		if n, _ := l.r.Read(one[:]); n > 0 {
			l.over = true
		}
		return 0, io.EOF
	}
	if int64(len(p)) > l.left {
		p = p[:l.left]
	}
	n, err := l.r.Read(p)
	l.left -= int64(n)
	return n, err
}

// CheckBlocking reports, for every view with blocking, whether name is
// blocked or allowed there, and by which rule or list. It looks names up
// the way named does: rules first, then the lists in order. The answer is
// for clients at the normal step; a list of a higher step that blocks the
// name says so (Match.Step), and clients at step off skip every list.
func (e *Engine) CheckBlocking(st *model.State, name string) ([]blocking.Match, error) {
	n, err := blocking.NormalizeName(name)
	if err != nil {
		return nil, errf(KindBadRequest, "%v", err)
	}
	var out []blocking.Match
	feeds := map[string]string{} // file -> matching entry ("" none)
	var cur *blocking.Match
	for _, z := range render.PolicyZones(st, e.cfg.Paths) {
		if cur == nil || cur.View != z.View {
			out = append(out, blocking.Match{View: z.View, Result: "none"})
			cur = &out[len(out)-1]
		}
		if cur.Result != "none" || z.IsGate() {
			continue // an earlier zone decided, or a gate (it never blocks)
		}
		if z.ListID == 0 {
			owner, act, ok := blocking.MatchRules(blocking.RuleEntries(render.ViewRules(st, z.ViewID)), n)
			if ok {
				cur.Entry, cur.RuleID = owner, ruleFor(st, z.ViewID, owner)
				cur.Result = map[string]string{blocking.ActionBlock: "blocked", blocking.ActionAllow: "allowed"}[act]
			}
			continue
		}
		entry, seen := feeds[z.File]
		if !seen {
			owner, ok, err := blocking.MatchFeed(z.File, n)
			if err != nil && !errors.Is(err, fs.ErrNotExist) {
				return nil, err
			}
			if ok {
				entry = owner
			}
			feeds[z.File] = entry
		}
		if entry != "" {
			b := st.Blocklists[z.ListID]
			cur.Result, cur.Entry, cur.List = "blocked", entry, b.Name
			if b.StepID != model.StepNormal {
				cur.Step = st.StepName(b.StepID)
			}
		}
	}
	return out, nil
}

// ruleFor finds the rule behind a rules-zone entry: the rule for the
// entry's name, or (for entries added so the most specific rule wins) the
// closest rule above it that covers the names below it.
func ruleFor(st *model.State, viewID int64, owner string) int64 {
	name := strings.TrimPrefix(owner, "*.")
	wild := name != owner
	var best *model.BlockRule
	for _, r := range st.SortedBlockRules() {
		if !slices.Contains(st.BlockingViews(r.ViewIDs), viewID) {
			continue
		}
		if r.Name == name && (!wild || !r.Exact) {
			return r.ID
		}
		if !r.Exact && strings.HasSuffix(name, "."+r.Name) && (best == nil || len(r.Name) > len(best.Name)) {
			best = r
		}
	}
	if best != nil {
		return best.ID
	}
	return 0
}