Prometheus metrics at /metrics, configured in jarl.yaml
New metrics: section (enabled, listen, allow, token). /metrics is served on the main listeners, the local socket (root and jarld's user) and optional metrics-only addresses; a scrape needs an allowed address or the bearer token. Exports health (named up, problems, drift, backups, blocklists, DNSSEC phases, token expiry) and, with stats on, named's and Jarl's query counters since jarld started plus named's process and cache gauges. docs/metrics.md has example queries and alerts checked with promtool. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adamandClaude Opus 5.5 committed
1 parent
7e6e274a89
commit
c37de31f4c
17 files changed
+1126
-4
No files matched your search
@@ -0,0 +1,206 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"jarl/internal/stats"
|
||||
)
|
||||
|
||||
// scrape GETs /metrics with an optional bearer token.
|
||||
func (e *apiEnv) scrape(token string) (string, int) {
|
||||
e.t.Helper()
|
||||
r, _ := http.NewRequest("GET", e.ts.URL+"/metrics", nil)
|
||||
if token != "" {
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
e.t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return string(b), resp.StatusCode
|
||||
}
|
||||
|
||||
var (
|
||||
promSampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*"(,[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*")*\})? -?[0-9.eE+-]+$`)
|
||||
promTypeRe = regexp.MustCompile(`^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (gauge|counter)$`)
|
||||
)
|
||||
|
||||
// checkExposition checks Prometheus's text format: valid lines, and each
|
||||
// metric's samples together under a single TYPE.
|
||||
func checkExposition(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
typed := map[string]bool{}
|
||||
cur := ""
|
||||
for _, l := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") {
|
||||
switch {
|
||||
case strings.HasPrefix(l, "# HELP "):
|
||||
case strings.HasPrefix(l, "# TYPE "):
|
||||
m := promTypeRe.FindStringSubmatch(l)
|
||||
if m == nil {
|
||||
t.Errorf("bad TYPE line %q", l)
|
||||
continue
|
||||
}
|
||||
if typed[m[1]] {
|
||||
t.Errorf("metric %s appears twice", m[1])
|
||||
}
|
||||
typed[m[1]], cur = true, m[1]
|
||||
default:
|
||||
if !promSampleRe.MatchString(l) {
|
||||
t.Errorf("bad sample line %q", l)
|
||||
}
|
||||
if name, _, _ := strings.Cut(strings.SplitN(l, " ", 2)[0], "{"); name != cur {
|
||||
t.Errorf("sample %q is not under its TYPE (%s)", l, cur)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAccess(t *testing.T) {
|
||||
loopback := []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("::1/128")}
|
||||
e := newAPIEnv(t)
|
||||
if _, status := e.scrape(""); status != 404 {
|
||||
t.Errorf("disabled = %d, want 404", status)
|
||||
}
|
||||
|
||||
e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: loopback}})
|
||||
body, status := e.scrape("")
|
||||
if status != 200 {
|
||||
t.Fatalf("allowed scrape = %d: %s", status, body)
|
||||
}
|
||||
checkExposition(t, body)
|
||||
for _, want := range []string{`jarl_build_info{version="test"} 1`, `jarl_problems{severity="error"} 0`, `jarl_objects{kind="zones"} 0`, "jarl_open_changesets 0", "jarl_metrics_state_ok 1"} {
|
||||
if !strings.Contains(body, want+"\n") {
|
||||
t.Errorf("missing %q in:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, "jarl_queries_total") {
|
||||
t.Error("query counters without statistics")
|
||||
}
|
||||
// No token is configured: one that is sent is refused.
|
||||
if _, status := e.scrape("whatever-token-123"); status != 401 {
|
||||
t.Errorf("token without metrics.token = %d, want 401", status)
|
||||
}
|
||||
|
||||
e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}}})
|
||||
if body, status := e.scrape(""); status != 403 || !strings.Contains(body, "metrics.allow") {
|
||||
t.Errorf("not allowed = %d %s", status, body)
|
||||
}
|
||||
|
||||
const token = "0123456789abcdef-metrics"
|
||||
e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Token: token}})
|
||||
if _, status := e.scrape(""); status != 401 {
|
||||
t.Errorf("no token = %d, want 401", status)
|
||||
}
|
||||
if _, status := e.scrape("wrong-token-0123456789"); status != 401 {
|
||||
t.Errorf("wrong token = %d, want 401", status)
|
||||
}
|
||||
if _, status := e.scrape(token); status != 200 {
|
||||
t.Errorf("right token = %d, want 200", status)
|
||||
}
|
||||
// The API's tokens don't count for /metrics.
|
||||
if _, status := e.scrape(e.token); status != 401 {
|
||||
t.Errorf("API token = %d, want 401", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsState(t *testing.T) {
|
||||
e := newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
|
||||
e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
|
||||
e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": "https://lists.example/ads.txt"}, 201)
|
||||
e.srv.metricsCache.body = nil // skip the cache
|
||||
body, _ := e.scrape("")
|
||||
checkExposition(t, body)
|
||||
for _, want := range []string{`jarl_objects{kind="zones"} 1`, `jarl_blocklist_enabled{list="ads"} 1`, "jarl_blocking_paused 0"} {
|
||||
if !strings.Contains(body, want+"\n") {
|
||||
t.Errorf("missing %q in:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
// Cached: a change shows after metricsStateTTL.
|
||||
e.do("POST", "/zones", e.token, map[string]any{"name": "example.org", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201)
|
||||
if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 1`+"\n") {
|
||||
t.Error("state was not cached")
|
||||
}
|
||||
e.srv.metricsCache.at = time.Now().Add(-metricsStateTTL)
|
||||
if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 2`+"\n") {
|
||||
t.Errorf("state not refreshed:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsStats(t *testing.T) {
|
||||
ss, err := stats.OpenStore(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ss.Close()
|
||||
opts := stats.Options{QueryLog: true, LogFile: "/nonexistent", StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour}
|
||||
c := stats.NewCollector(ss, opts, func() map[string][]string { return map[string][]string{"internal": {"example.com"}} }, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
ts := time.Now().Format("02-Jan-2006 15:04:05.000")
|
||||
c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5353 (www.example.com): view internal: response: www.example.com IN A NOERROR 1 0 1 +E(0)K (10.10.0.2)")
|
||||
c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid")
|
||||
c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: response: ads.example IN A NXDOMAIN 1 0 1 +E(0)K (10.10.0.2)")
|
||||
|
||||
e := newAPIEnvConfig(t, Config{Version: "test", Stats: c, Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}})
|
||||
body, status := e.scrape("")
|
||||
if status != 200 {
|
||||
t.Fatalf("scrape = %d", status)
|
||||
}
|
||||
checkExposition(t, body)
|
||||
for _, want := range []string{
|
||||
"jarl_queries_total 0", "jarl_logged_responses_total 2", `jarl_responses_by_view_total{view="internal"} 2`,
|
||||
`jarl_responses_by_zone_total{zone="example.com"} 1`, `jarl_responses_by_zone_total{zone="-"} 1`,
|
||||
"jarl_blocked_queries_total 1", `jarl_blocked_by_list_total{list="hagezi"} 1`, "jarl_stats_log_lines_total 2",
|
||||
} {
|
||||
if !strings.Contains(body, want+"\n") {
|
||||
t.Errorf("missing %q in:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsMetricsMapping(t *testing.T) {
|
||||
pw := &promWriter{}
|
||||
totals := map[string]int64{"queries": 10, "rcode:NOERROR": 8, "rcode:NXDOMAIN": 2, "qtype:A": 10, "ratelimit:slipped": 3,
|
||||
"cache:hits": 4, "rtt:<10ms": 5, "unknown:x": 1, "rcode": 7}
|
||||
srv := &stats.Server{BootTime: "2026-10-07T20:00:00.000Z", Version: "9.20.4", PID: 42, RSS: 1 << 20, CPUSeconds: 1.5, SocketsUDP: 4}
|
||||
statsMetrics(pw, totals, srv, stats.Status{})
|
||||
var buf bytes.Buffer
|
||||
pw.writeTo(&buf)
|
||||
out := buf.String()
|
||||
checkExposition(t, out)
|
||||
for _, want := range []string{
|
||||
"jarl_queries_total 10", `jarl_responses_by_rcode_total{rcode="NXDOMAIN"} 2`, `jarl_ratelimit_responses_total{action="slipped"} 3`,
|
||||
`jarl_upstream_responses_by_rtt_total{rtt="<10ms"} 5`, `jarl_cache_lookups_total{result="hits"} 4`,
|
||||
"jarl_named_start_time_seconds 1791403200", "jarl_named_cpu_seconds_total 1.5", `jarl_named_sockets{proto="udp"} 4`,
|
||||
"jarl_named_resident_memory_bytes 1048576",
|
||||
} {
|
||||
if !strings.Contains(out, want+"\n") {
|
||||
t.Errorf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "unknown") || strings.Contains(out, "jarl_responses_by_rcode_total 7") {
|
||||
t.Errorf("unknown keys exported:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromWriterEscaping(t *testing.T) {
|
||||
pw := &promWriter{}
|
||||
pw.gauge("m", "a \\ help\nline", 1, "l", "a\"b\\c\nd")
|
||||
var buf bytes.Buffer
|
||||
pw.writeTo(&buf)
|
||||
want := "# HELP m a \\\\ help\\nline\n# TYPE m gauge\nm{l=\"a\\\"b\\\\c\\nd\"} 1\n"
|
||||
if buf.String() != want {
|
||||
t.Errorf("got %q, want %q", buf.String(), want)
|
||||
}
|
||||
if got := formatFloat(1759867200.123); got != "1759867200.123" {
|
||||
t.Errorf("formatFloat = %s", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user