package stats import ( "context" "fmt" "strings" "time" ) // Exploring the query log (docs/stats.md#exploring): every answer of the // last days (stats.query_retention), filtered by any mix of client, name, // view, type, result and blocking, broken down by each of them. // ExploreFilter narrows the query log. Empty fields don't filter. type ExploreFilter struct { Client string // a client key, as the top lists show it Name string // exactly this name Domain string // this name or any below it View string Type string RCode string // Blocked is "yes" (blocked by anything), "no", or a blocklist's name // ("rules" for your own rules). Blocked string Since time.Duration // how far back (default 24h) } // Dimensions are what the query log breaks down by, in the order shown. var Dimensions = []string{"client", "name", "view", "type", "rcode", "blocked"} // Exploration is what the query log says for a filter. type Exploration struct { From time.Time `json:"from"` To time.Time `json:"to"` Total int64 `json:"total"` // Blocked counts the blocked answers among them. Blocked int64 `json:"blocked"` // Series are the answers (and blocked ones) per Step seconds. Step int64 `json:"step"` Series []ExplorePoint `json:"series"` // By are the top values of each dimension the filter leaves open. By map[string][]Entry `json:"by"` // Recent are the latest answers. Recent []QueryRow `json:"recent"` // Oldest is the first answer the query log still has. Oldest *time.Time `json:"oldest,omitempty"` // Retention is how long it keeps them. Retention string `json:"retention"` } // ExplorePoint is one step of the series. type ExplorePoint struct { Time int64 `json:"t"` Count int64 `json:"count"` Blocked int64 `json:"blocked"` } // ErrNoQueryLog is returned when the query log is off. var ErrNoQueryLog = fmt.Errorf("the query log is off: it needs stats.query_log and a stats.query_retention above 0 in /etc/jarl/jarl.yaml (docs/stats.md#exploring)") // where builds the SQL condition and arguments for a filter. func (q *Querier) where(f ExploreFilter, from int64) (string, []any) { conds := []string{"ts >= ?"} args := []any{from} eq := func(col, v string) { if v != "" { conds = append(conds, col+" = ?") args = append(args, v) } } eq("client", f.Client) eq("name", strings.ToLower(strings.TrimSuffix(f.Name, "."))) eq("view", f.View) eq("type", strings.ToUpper(f.Type)) eq("rcode", strings.ToUpper(f.RCode)) if d := strings.ToLower(strings.TrimSuffix(f.Domain, ".")); d != "" { conds = append(conds, "(name = ? OR (length(name) > length(?) AND substr(name, length(name) - length(?)) = ?))") args = append(args, d, d, d, "."+d) } switch f.Blocked { case "": case "yes": conds = append(conds, "blocked != ''") case "no": conds = append(conds, "blocked = ''") default: eq("blocked", f.Blocked) } if len(q.Views) > 0 { marks := strings.TrimSuffix(strings.Repeat("?,", len(q.Views)), ",") conds = append(conds, "view IN ("+marks+")") for _, v := range q.Views { args = append(args, v) } } return strings.Join(conds, " AND "), args } // Explore answers a filter: totals, a series, the top values of each open // dimension (topN each) and the latest answers (recent of them). func (q *Querier) Explore(ctx context.Context, f ExploreFilter, topN, recent int) (*Exploration, error) { if !q.Opts.QueryLog || q.Opts.RetainQueries <= 0 { return nil, ErrNoQueryLog } now := q.Now() since := f.Since if since <= 0 { since = 24 * time.Hour } if since > q.Opts.RetainQueries { since = q.Opts.RetainQueries } from := now.Add(-since).Unix() out := &Exploration{From: time.Unix(from, 0), To: now, By: map[string][]Entry{}, Series: []ExplorePoint{}, Recent: []QueryRow{}, Retention: humanDuration(q.Opts.RetainQueries)} cond, args := q.where(f, from) db := q.Store.db if err := db.QueryRowContext(ctx, `SELECT count(*), coalesce(sum(blocked != ''), 0) FROM queries WHERE `+cond, args...).Scan(&out.Total, &out.Blocked); err != nil { return nil, err } // The series: about 100 steps over the range, in whole minutes. step := int64(since.Seconds()) / 100 step = max(60, step-step%60) out.Step = step rows, err := db.QueryContext(ctx, `SELECT ts - ts % ? AS t, count(*), sum(blocked != '') FROM queries WHERE `+cond+` GROUP BY t ORDER BY t`, append([]any{step}, args...)...) if err != nil { return nil, err } got := map[int64]ExplorePoint{} for rows.Next() { var p ExplorePoint if err := rows.Scan(&p.Time, &p.Count, &p.Blocked); err != nil { rows.Close() return nil, err } got[p.Time] = p } rows.Close() for t := from - from%step; t <= now.Unix(); t += step { p := got[t] p.Time = t out.Series = append(out.Series, p) } open := map[string]bool{"client": f.Client == "", "name": f.Name == "", "view": f.View == "", "type": f.Type == "", "rcode": f.RCode == "", "blocked": f.Blocked == "" || f.Blocked == "yes"} for _, dim := range Dimensions { if !open[dim] { continue } c := cond if dim == "blocked" { c += " AND blocked != ''" } rows, err := db.QueryContext(ctx, `SELECT `+dim+`, count(*) AS n FROM queries WHERE `+c+` GROUP BY `+dim+` ORDER BY n DESC, `+dim+` LIMIT ?`, append(args, topN)...) if err != nil { return nil, err } es := []Entry{} for rows.Next() { var e Entry if err := rows.Scan(&e.Key, &e.Count); err != nil { rows.Close() return nil, err } es = append(es, e) } rows.Close() out.By[dim] = es } rows, err = db.QueryContext(ctx, `SELECT ts, client, view, name, type, rcode, blocked FROM queries WHERE `+cond+` ORDER BY ts DESC, rowid DESC LIMIT ?`, append(args, recent)...) if err != nil { return nil, err } for rows.Next() { var r QueryRow if err := rows.Scan(&r.Time, &r.Client, &r.View, &r.Name, &r.Type, &r.RCode, &r.Blocked); err != nil { rows.Close() return nil, err } out.Recent = append(out.Recent, r) } rows.Close() var oldest *int64 if err := db.QueryRowContext(ctx, `SELECT min(ts) FROM queries`).Scan(&oldest); err == nil && oldest != nil { t := time.Unix(*oldest, 0) out.Oldest = &t } return out, nil } // QueryLogSize returns how many answers the query log has. func (s *Store) QueryLogSize(ctx context.Context) (int64, error) { var n int64 err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM queries`).Scan(&n) return n, err }