diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce1d5b..81f629e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ All notable changes to Jarl are documented here. The format follows ## [Unreleased] +### Added + +- Prometheus metrics at `/metrics` ([docs/metrics.md](docs/metrics.md)), + set up in the new `metrics:` section of jarl.yaml (off by default). + `metrics.allow` lists the addresses that may scrape (loopback by + default), `metrics.token` lets anyone who sends it as a bearer token + scrape, and `metrics.listen` adds addresses that serve only `/metrics` + (for a Prometheus server elsewhere). Root and jarld's own user can also + scrape on the local socket. + - Health, always: named up, recursive and TCP clients against their + limits, adopted, open changesets, last apply, validation problems by + severity, drifted files, object counts, the last backup (and failure, + and whether it is stale), blocking paused, each blocklist's entries, + last check and change and whether it fails, the DNSSEC phase of every + signed zone, and when API tokens expire. + - With query statistics on: named's counters (queries, result codes, + record types, UDP/TCP, authoritative/recursive, rate limiting, upstream + round-trip times, DNSSEC validation, cache hits) and Jarl's (logged + responses per view and zone, blocked queries per blocklist), as + counters since jarld started, plus named's memory, CPU, threads, cache + and sockets, and the collector's health. + - Example queries and alert rules, checked with promtool. + ## [0.4.0] - 2026-10-11 ### Upgrading from 0.3 diff --git a/README.md b/README.md index bb3c30c..3e8e340 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,11 @@ error: www.adbraun.lol has a CNAME, so it can't have any other records (found A) queries, record types, cache hit rate. Shown in the web UI and the CLI, and exportable as JSON. Stored as aggregates in a separate database with retention you set. +* **Prometheus metrics** (optional): health to alert on (named down, + configuration problems, drift, failing blocklists, stale backups, DNSSEC + steps waiting at the parent, expiring tokens) and query counters per + result code, view, zone and blocklist, with example alerts. Set up in + jarl.yaml, with an address allow list and/or a bearer token. * **Blocking** like Pi-hole, done by named itself: blocklists (Adblock, hosts or plain domains) downloaded and kept up to date, your own block and allow rules that beat every list, per view, with "why is this @@ -120,6 +125,7 @@ in `/etc/jarl/jarl.yaml` to reach it from your management network. | [docs/dnssec.md](docs/dnssec.md) | DNSSEC signing, going public with your own name servers, Porkbun | | [docs/acme.md](docs/acme.md) | Let's Encrypt DNS-01 with certbot, acme.sh, lego and Traefik | | [docs/stats.md](docs/stats.md) | Query statistics: enabling, what is collected, retention | +| [docs/metrics.md](docs/metrics.md) | Prometheus metrics: access, what is exported, example alerts | | [docs/blocking.md](docs/blocking.md) | Blocking ads, trackers and malware: blocklists, your own rules, views, memory | | [docs/validation.md](docs/validation.md) | Every check Jarl makes | | [docs/operations.md](docs/operations.md) | Backups, drift, failures, recovery, monitoring | diff --git a/cmd/jarld/main.go b/cmd/jarld/main.go index d3f583c..30e5100 100644 --- a/cmd/jarld/main.go +++ b/cmd/jarld/main.go @@ -303,6 +303,11 @@ func serveCmd() *cobra.Command { a.log.Warn("the configuration file holds the Porkbun API key and everyone can read it; make it readable only by root and jarld (chown root:jarl, chmod 640)", "file", configPath) } } + if a.cfg.Metrics.Enabled && a.cfg.Metrics.Token != "" { + if fi, err := os.Stat(configPath); err == nil && fi.Mode().Perm()&0o004 != 0 { + a.log.Warn("the configuration file holds metrics.token and everyone can read it; make it readable only by root and jarld (chown root:jarl, chmod 640)", "file", configPath) + } + } srv := server.New(server.Config{ Listen: a.cfg.Listen, TLSCertFile: a.cfg.TLS.CertFile, TLSKeyFile: a.cfg.TLS.KeyFile, TrustedProxies: a.cfg.Proxies(), SessionLifetime: time.Duration(a.cfg.Session.Lifetime), @@ -311,6 +316,8 @@ func serveCmd() *cobra.Command { BackupDir: a.cfg.Backup.Dir, BackupWarnAfter: time.Duration(a.cfg.Backup.WarnAfter), Porkbun: pb, PorkbunAllows: a.cfg.Porkbun.Allows, TOTPRequire: a.cfg.TOTP.Require, TOTPIssuer: a.cfg.TOTP.IssuerName(), + Metrics: server.Metrics{Enabled: a.cfg.Metrics.Enabled, Listen: a.cfg.Metrics.Listen, + Allow: a.cfg.Metrics.AllowPrefixes(), Token: a.cfg.Metrics.Token}, HTTPReq: server.HTTPReq{ Enabled: a.cfg.ACME.HTTPReq.Enabled, AllowPlainHTTP: !a.cfg.ACME.HTTPReq.RequireHTTPS, diff --git a/docs/api.md b/docs/api.md index c0bb6fb..8571c5d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -10,6 +10,8 @@ what is documented here. (`www.adbraun.lol.`), and display helpers drop it (`display_name`, `relative_name`). * `GET /healthz` (no authentication) returns `{"ok": true, "version": ...}`. +* `GET /metrics` serves Prometheus metrics when `metrics.enabled` is set. + Access is set in jarl.yaml, not with API tokens ([metrics.md](metrics.md)). ## Authentication diff --git a/docs/configuration.md b/docs/configuration.md index 8a0f48a..017a476 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,18 @@ The `stats:` section turns on query statistics (off by default). See [stats.md](stats.md) for every setting and what it changes in named's configuration. Changing it takes effect when jarld restarts. +### Prometheus metrics + +| Key | Default | Meaning | +|-----|---------|---------| +| `metrics.enabled` | `false` | Serve Prometheus metrics at `/metrics` on `listen` and the local socket ([metrics.md](metrics.md)). | +| `metrics.listen` | `[]` | Extra `ADDRESS:PORT`s that serve only `/metrics`, over plain HTTP, e.g. `["0.0.0.0:9450"]`. | +| `metrics.allow` | `["127.0.0.1", "::1"]` | Addresses or prefixes that may scrape without a token. | +| `metrics.token` | `""` | A bearer token (at least 16 characters) that lets any address scrape. Keep jarl.yaml readable by root and jarld only when set. | + +Query counters need `stats.enabled` as well. Changes take effect when jarld +restarts. + ### ACME (lego httpreq endpoint) An endpoint for lego's `httpreq` DNS provider, so lego and Traefik can diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..f031a2c --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,231 @@ +# Prometheus metrics + +jarld can serve metrics for [Prometheus](https://prometheus.io/) at +`/metrics`: +- **health** you can alert on: named up, validation problems, files changed + outside Jarl, blocklists that fail to download, backup age, DNSSEC zones + waiting for you at the parent, and API tokens about to expire, +- **queries**, once [query statistics](stats.md) are on: named's counters + (result codes, record types, cache, upstream times, DNSSEC validation, rate + limiting) and Jarl's own (responses per view and zone, blocked queries per + blocklist), plus named's memory, CPU, cache and sockets. + +Metrics are **off by default**. Everything is set in `/etc/jarl/jarl.yaml`; +nothing is stored in the database. + +## Turning it on + +Prometheus on the same server needs only: + +```yaml +metrics: + enabled: true +``` + +```sh +sudo systemctl restart jarl +curl -s http://127.0.0.1:8053/metrics | head +``` + +`/metrics` is served on the addresses in `listen` (and their TLS, if set) +and on the local socket. Query counters also need `stats.enabled: true` +([stats.md](stats.md)); without it you get only the health metrics. + +### Prometheus on another machine + +Give `/metrics` an address of its own and say who may scrape it: + +```yaml +metrics: + enabled: true + listen: ["0.0.0.0:9450"] # serves only /metrics, over plain HTTP + allow: ["192.168.1.20"] # the Prometheus server +``` + +```yaml +# prometheus.yml +scrape_configs: + - job_name: jarl + static_configs: + - targets: ["dns.lan:9450"] +``` + +Or use a token instead of (or as well as) the address list. Any address +that sends it may scrape: + +```yaml +metrics: + enabled: true + listen: ["0.0.0.0:9450"] + token: "…" # e.g. from: openssl rand -hex 24 +``` + +```yaml +# prometheus.yml +scrape_configs: + - job_name: jarl + authorization: + credentials: "…" # or credentials_file: /etc/prometheus/jarl.token + static_configs: + - targets: ["dns.lan:9450"] +``` + +With a token in jarl.yaml, keep the file readable by root and jarld only +(`chown root:jarl /etc/jarl/jarl.yaml; chmod 640 /etc/jarl/jarl.yaml`); +jarld warns at start when everyone can read it. On `metrics.listen` the +token travels unencrypted, so use it on a network you trust, or scrape the +main listener over HTTPS instead (`scheme: https` and its port in +Prometheus). + +### Settings + +| Key | Default | Meaning | +|-----|---------|---------| +| `metrics.enabled` | `false` | Serve `/metrics`. | +| `metrics.listen` | `[]` | Extra `ADDRESS:PORT`s that serve only `/metrics`, over plain HTTP. | +| `metrics.allow` | `["127.0.0.1", "::1"]` | Addresses or prefixes that may scrape without a token. Behind a reverse proxy in `trusted_proxies`, this is the client's address from `X-Forwarded-For`. `[]` with a token: the token is required everywhere. | +| `metrics.token` | `""` | A bearer token (at least 16 characters) that lets any address scrape. | + +A scrape is allowed when it comes from an address in `allow`, **or** +carries the token, **or** comes over the local socket from root or jarld's +own user. A wrong token is refused (and counts towards the per-address +limit on failed logins). Jarl's API tokens don't work for `/metrics`. + +The firewall must let Prometheus reach the port. `allow` is checked by +jarld, not the kernel. + +## What is exported + +Everything is prefixed `jarl_`. The text format is checked with `promtool +check metrics`. + +### Health (always) + +These are read at most every 15 seconds; a scrape in between gets the +same values. + +| Metric | Meaning | +|--------|---------| +| `jarl_build_info{version}` | Always 1. | +| `jarl_start_time_seconds` | When jarld started. | +| `jarl_metrics_state_ok` | 0 when Jarl's state couldn't be read (see jarld's log). | +| `jarl_adopted` | 1 when named loads Jarl's configuration. | +| `jarl_named_up` | 1 when named answers `rndc status` (missing with `bind.no_reload`). | +| `jarl_named_recursive_clients`, `…_soft_limit`, `…_limit` | Recursive clients now, and named's limits (`rndc status`). | +| `jarl_named_tcp_clients`, `…_limit` | TCP clients now, and the limit. | +| `jarl_open_changesets` | Changesets with staged changes not applied yet. | +| `jarl_last_apply_timestamp_seconds` | When the last changeset was applied. | +| `jarl_problems{severity}` | Validation problems (`error`, `warning`, `info`) as on the dashboard, including named's own warnings. | +| `jarl_drifted_files` | Managed files changed or removed outside Jarl. | +| `jarl_objects{kind}` | Views, zones, records, ACLs, TSIG keys and options. | +| `jarl_backup_last_success_timestamp_seconds`, `jarl_backup_last_size_bytes` | The newest backup by `jarld backup`. | +| `jarl_backup_last_failure_timestamp_seconds` | A failed backup newer than the last good one. | +| `jarl_backup_stale` | 1 when the newest backup is older than `backup.warn_after`. | +| `jarl_blocking_paused` | 1 while blocking is paused (only with blocklists or rules). | +| `jarl_blocking_paused_until_timestamp_seconds` | When a timed pause ends. | +| `jarl_blocklist_enabled{list}` | 1 for lists that are switched on. | +| `jarl_blocklist_entries{list}` | Domains in the last good download. | +| `jarl_blocklist_failing{list}` | 1 when the last download or check of an enabled list failed. | +| `jarl_blocklist_last_check_timestamp_seconds{list}`, `jarl_blocklist_last_change_timestamp_seconds{list}` | When the list was last checked, and last changed. | +| `jarl_dnssec_phase{zone,view,phase}` | Always 1, one per signed zone; `phase` is one of the [DNSSEC phases](dnssec.md). `ready` means: add the DS at the parent. `removing_ds`: remove it. | +| `jarl_token_expiry_timestamp_seconds{id,user,token}` | When each API token with an expiry date expires. | + +### Queries (with `stats.enabled`) + +The counters start at zero when jarld starts (Prometheus handles that). +named's counters move once a minute, when Jarl polls named's statistics +channel. Counters from the response log (`responses`, views, zones, +blocking) move as lines are read and need `stats.query_log`. Clients in +`stats.exclude_clients` are left out of those. Names and clients are not +exported, because there would be too many of them. Use the Statistics page +for them. + +| Metric | Meaning | +|--------|---------| +| `jarl_queries_total` | Queries named received. | +| `jarl_responses_by_rcode_total{rcode}` | Responses by result code (`NOERROR`, `NXDOMAIN`, `SERVFAIL`, `REFUSED`, …). | +| `jarl_queries_by_type_total{type}` | Queries by record type. | +| `jarl_queries_by_transport_total{transport}` | `udp` and `tcp`. | +| `jarl_answers_total{source}` | `authoritative` or `recursive`. | +| `jarl_recursion_queries_total`, `jarl_dropped_queries_total`, `jarl_duplicate_queries_total` | Queries that caused recursion, that named dropped, and duplicates. | +| `jarl_ratelimit_responses_total{action}` | Responses rate limiting `dropped` or `slipped`. | +| `jarl_upstream_responses_by_rtt_total{rtt}` | Upstream answers during recursion by round-trip time (`<10ms` … `>1600ms`). | +| `jarl_dnssec_validations_total{result}` | `valid`, `valid-negative` and `failed`. | +| `jarl_cache_lookups_total{result}` | `hits` and `misses`. | +| `jarl_logged_responses_total` | Responses read from the response log. | +| `jarl_responses_by_view_total{view}` | Logged responses per view (`_default` without views). | +| `jarl_responses_by_zone_total{zone}` | Logged responses per local zone; `-` for names in no local zone. | +| `jarl_blocked_queries_total` | Queries blocking answered. | +| `jarl_blocked_by_list_total{list}` | Blocked queries per blocklist; `rules` for your own rules. | +| `jarl_stats_poll_ok`, `jarl_stats_last_poll_timestamp_seconds` | Whether the last poll of named's statistics channel worked, and when it was. | +| `jarl_stats_log_lines_total`, `jarl_stats_last_log_line_timestamp_seconds` | Response log lines read, and the time of the last one. | +| `jarl_stats_warnings` | Warnings on the Statistics page (e.g. no log lines while named answers queries). | +| `jarl_named_start_time_seconds`, `jarl_named_config_time_seconds` | When named started, and last loaded its configuration. | +| `jarl_named_resident_memory_bytes`, `jarl_named_threads`, `jarl_named_cpu_seconds_total` | named's process (`/proc`). | +| `jarl_named_memory_in_use_bytes` | Memory named counts as in use. | +| `jarl_named_cache_rrsets`, `jarl_named_cache_memory_bytes` | named's caches, all views together. | +| `jarl_named_sockets{proto}` | Open UDP and TCP sockets. | +| `jarl_named_recursive_clients_highwater`, `jarl_named_tcp_clients_highwater` | The most clients at once since named started. | + +Host metrics (load, memory, disks) are left to +[node_exporter](https://github.com/prometheus/node_exporter). + +## Example queries + +```promql +# Queries per second, and the share that failed +rate(jarl_queries_total[5m]) +rate(jarl_responses_by_rcode_total{rcode="SERVFAIL"}[5m]) / ignoring(rcode) rate(jarl_queries_total[5m]) + +# Blocked share of logged responses, per list +rate(jarl_blocked_by_list_total[1h]) / ignoring(list) group_left rate(jarl_logged_responses_total[1h]) + +# Busiest zones +topk(5, rate(jarl_responses_by_zone_total{zone!="-"}[1h])) + +# Cache hit rate +rate(jarl_cache_lookups_total{result="hits"}[5m]) / ignoring(result) sum without(result) (rate(jarl_cache_lookups_total[5m])) +``` + +## Example alerts + +```yaml +groups: + - name: jarl + rules: + - alert: JarlDown + expr: up{job="jarl"} == 0 + for: 5m + - alert: NamedDown + expr: jarl_named_up == 0 + for: 2m + - alert: JarlConfigErrors + expr: jarl_problems{severity="error"} > 0 + for: 15m + - alert: JarlDrift + expr: jarl_drifted_files > 0 + for: 15m + - alert: BlocklistFailing + expr: jarl_blocklist_failing == 1 + for: 6h + - alert: BackupStale + expr: jarl_backup_stale == 1 + - alert: DNSSECWaitingForYou + expr: jarl_dnssec_phase{phase=~"ready|removing_ds|unsigned"} == 1 + for: 1h + annotations: + summary: "{{ $labels.zone }}: DNSSEC is waiting for a step at the parent ({{ $labels.phase }})" + - alert: TokenExpiresSoon + expr: jarl_token_expiry_timestamp_seconds - time() < 7 * 86400 + annotations: + summary: "API token {{ $labels.token }} of {{ $labels.user }} expires within a week" + - alert: BlockingPausedForLong + expr: jarl_blocking_paused == 1 + for: 24h + - alert: NamedRecursiveClientsHigh + expr: jarl_named_recursive_clients / jarl_named_recursive_clients_soft_limit > 0.8 + for: 5m + - alert: StatsNotCollecting + expr: jarl_stats_poll_ok == 0 or jarl_stats_warnings > 0 + for: 15m +``` diff --git a/docs/operations.md b/docs/operations.md index 6fca561..194d60f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -363,6 +363,9 @@ rndc reconfig ## Monitoring * `GET /healthz` (no authentication) reports that jarld is up. +* Prometheus: `/metrics` covers named up, configuration problems, drift, + blocklist failures, backup age, DNSSEC steps waiting at the parent and + expiring tokens, with example alerts ([metrics.md](metrics.md)). * On the server, a check can use the local socket and needs no token: `sudo jarl status -o json`. * `GET /api/v1/status` with a read-only token reports named reachability, diff --git a/docs/stats.md b/docs/stats.md index 6bd4b02..2057ca6 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -175,6 +175,9 @@ jarl stats summary --since 30d -o json # export (also -o yaml) **API**: see [api.md](api.md#query-statistics). +**Prometheus**: with `metrics.enabled`, the counters are also served at +`/metrics` ([metrics.md](metrics.md)). + **Permissions**: the `stats` resource (`stats:read`). The `readonly` and `operator` roles include it, `dns-editor` doesn't. Statistics show client addresses and what they look up, so grant it with care. diff --git a/internal/config/config.go b/internal/config/config.go index 5b1867c..97a0d56 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ package config import ( "bytes" "fmt" + "net" "net/netip" "net/url" "os" @@ -38,6 +39,56 @@ type Config struct { ACME ACME `yaml:"acme"` Porkbun Porkbun `yaml:"porkbun"` Blocking Blocking `yaml:"blocking"` + Metrics Metrics `yaml:"metrics"` +} + +// Metrics configures the Prometheus endpoint /metrics (docs/metrics.md). +// A scrape is allowed from an address in Allow, or with Token as a bearer +// token; root and jarld's own user may also scrape on the local socket. +type Metrics struct { + Enabled bool `yaml:"enabled"` + // Listen are extra addresses that serve only /metrics, over plain + // HTTP. /metrics is always served on the main listeners too. + Listen []string `yaml:"listen"` + // Allow are the addresses or prefixes that may scrape without a token. + Allow []string `yaml:"allow"` + // Token, when set, lets any address that sends it as + // "Authorization: Bearer TOKEN" scrape. + Token string `yaml:"token"` +} + +func (m *Metrics) check() error { + if !m.Enabled { + return nil + } + for _, a := range m.Listen { + if _, port, err := net.SplitHostPort(a); err != nil || port == "" { + return fmt.Errorf("listen %q must be ADDRESS:PORT, e.g. 0.0.0.0:9450", a) + } + } + for _, p := range m.Allow { + if _, err := ParsePrefix(p); err != nil { + return fmt.Errorf("allow: %v", err) + } + } + if m.Token != "" && (len(m.Token) < 16 || strings.ContainsAny(m.Token, " \t\r\n")) { + return fmt.Errorf("token must be at least 16 characters, without spaces (e.g. from 'openssl rand -hex 24')") + } + if len(m.Allow) == 0 && m.Token == "" { + return fmt.Errorf("set allow, token or both, or nobody can scrape") + } + return nil +} + +// AllowPrefixes returns the parsed allow list. +func (m *Metrics) AllowPrefixes() []netip.Prefix { + var out []netip.Prefix + for _, p := range m.Allow { + if pf, err := ParsePrefix(p); err == nil { + out = append(out, pf.Masked()) + } + } + return out } // Blocking configures blocklist downloads (docs/blocking.md). The lists @@ -323,6 +374,7 @@ func Default() *Config { }, ACME: ACME{HTTPReq: HTTPReq{RequireHTTPS: true}}, Blocking: Blocking{Refresh: Duration(24 * time.Hour), MaxDownloadMB: 64, MaxEntries: 1_000_000}, + Metrics: Metrics{Allow: []string{"127.0.0.1", "::1"}}, Backup: Backup{Dir: "/var/backups/jarl", Retention: Duration(30 * 24 * time.Hour), KeepAtLeast: 7, WarnAfter: Duration(2 * 24 * time.Hour)}, } } @@ -394,6 +446,9 @@ func (c *Config) check() error { if err := c.Blocking.check(); err != nil { return fmt.Errorf("blocking: %w", err) } + if err := c.Metrics.check(); err != nil { + return fmt.Errorf("metrics: %w", err) + } switch c.Log.Format { case "text", "json": default: @@ -633,4 +688,23 @@ blocking: refresh: 24h # check each list for changes this often (at least 1h) max_download_mb: 64 # refuse larger downloads max_entries: 1000000 # refuse lists with more domains + +# Prometheus metrics at /metrics (docs/metrics.md): named's counters and +# Jarl's own (blocking, zones, views), plus health gauges to alert on +# (named up, validation problems, blocklist errors, backup age, DNSSEC +# steps, token expiry). Off by default. /metrics is served on the +# addresses in 'listen' at the top of this file; a scrape is allowed from +# an address in 'allow' below, or with the token. Query counters need +# stats.enabled. +metrics: + enabled: false + # Extra addresses that serve only /metrics, over plain HTTP, e.g. + # ["0.0.0.0:9450"] for a Prometheus server elsewhere on the network. + listen: [] + # Addresses or prefixes that may scrape without a token. + allow: ["127.0.0.1", "::1"] + # A bearer token (at least 16 characters) that lets any address scrape: + # in Prometheus, 'authorization: { credentials: TOKEN }'. Keep this file + # readable by root and jarld only when you set it. + token: "" ` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 61d31d8..9b0af23 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -87,3 +87,31 @@ func TestStatsConfig(t *testing.T) { } } } + +func TestMetricsConfig(t *testing.T) { + d := Default() + if d.Metrics.Enabled || len(d.Metrics.AllowPrefixes()) != 2 || d.Metrics.Token != "" { + t.Errorf("defaults = %+v", d.Metrics) + } + path := filepath.Join(t.TempDir(), "jarl.yaml") + for _, tc := range []struct { + yaml string + ok bool + }{ + {"metrics: {enabled: true}", true}, + {"metrics: {enabled: true, listen: [\"0.0.0.0:9450\", \"[::]:9450\"], allow: [10.0.0.5, 192.168.1.0/24]}", true}, + {"metrics: {enabled: true, allow: [], token: 0123456789abcdef}", true}, + {"metrics: {enabled: true, allow: []}", false}, // nobody could scrape + {"metrics: {enabled: true, token: short}", false}, + {"metrics: {enabled: true, token: \"has space 0123456789\"}", false}, + {"metrics: {enabled: true, listen: [\"9450\"]}", false}, + {"metrics: {enabled: true, allow: [prometheus.lan]}", false}, + {"metrics: {enabled: false, listen: [nonsense]}", true}, // only checked when enabled + } { + os.WriteFile(path, []byte(tc.yaml+"\n"), 0o644) + _, err := Load(path, false) + if (err == nil) != tc.ok { + t.Errorf("%s: err = %v, want ok=%v", tc.yaml, err, tc.ok) + } + } +} diff --git a/internal/server/metrics.go b/internal/server/metrics.go new file mode 100644 index 0000000..49cab92 --- /dev/null +++ b/internal/server/metrics.go @@ -0,0 +1,474 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/subtle" + "errors" + "fmt" + "log/slog" + "math" + "net" + "net/http" + "net/netip" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "jarl/internal/dnssec" + "jarl/internal/engine" + "jarl/internal/model" + "jarl/internal/render" + "jarl/internal/stats" + "jarl/internal/validate" +) + +// Metrics configures the Prometheus endpoint /metrics (docs/metrics.md). +type Metrics struct { + Enabled bool + // Listen are extra addresses serving only /metrics (plain HTTP). + Listen []string + // Allow may scrape without a token; Token, when set, lets anyone who + // sends it scrape. + Allow []netip.Prefix + Token string +} + +// metricsStateTTL is how long the Jarl state part of /metrics is reused: +// it runs rndc status, validation and the drift check, too much for every +// scrape when several Prometheus servers scrape often. +const metricsStateTTL = 15 * time.Second + +// metricsCache holds the last Jarl state part of /metrics. +type metricsCache struct { + mu sync.Mutex + at time.Time + body []byte +} + +const metricsContentType = "text/plain; version=0.0.4; charset=utf-8" + +// metrics serves GET /metrics in Prometheus's text format. +func (s *Server) metrics(w http.ResponseWriter, r *http.Request) { + if !s.cfg.Metrics.Enabled { + http.Error(w, "metrics are disabled; set metrics.enabled: true in /etc/jarl/jarl.yaml and restart jarl (see docs/metrics.md)", http.StatusNotFound) + return + } + if code, msg := s.metricsAccess(r); code != 0 { + if code == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Bearer realm="jarl metrics"`) + } + http.Error(w, msg, code) + return + } + var buf bytes.Buffer + buf.Write(s.stateMetrics(r.Context())) + if c := s.cfg.Stats; c != nil { + pw := &promWriter{} + statsMetrics(pw, c.Totals(), c.Server(), c.Status()) + pw.writeTo(&buf) + } + w.Header().Set("Content-Type", metricsContentType) + w.Header().Set("Cache-Control", "no-store") + w.Write(buf.Bytes()) +} + +// metricsAccess decides whether a scrape is allowed: from an address in +// metrics.allow, with metrics.token, or on the local socket from root or +// jarld's own user. It returns 0 when allowed, or the status and message +// to refuse with. +func (s *Server) metricsAccess(r *http.Request) (int, string) { + m := s.cfg.Metrics + ip := s.clientIP(r) + if h := r.Header.Get("Authorization"); h != "" { + scheme, tok, _ := strings.Cut(h, " ") + if m.Token == "" || !strings.EqualFold(scheme, "Bearer") { + return http.StatusUnauthorized, "this server takes no token for /metrics (metrics.token is not set)" + } + if s.limiter.blocked("ip:"+ip, failsPerIP) { + return http.StatusTooManyRequests, "too many failed attempts; try again later" + } + got, want := sha256.Sum256([]byte(strings.TrimSpace(tok))), sha256.Sum256([]byte(m.Token)) + if subtle.ConstantTimeCompare(got[:], want[:]) != 1 { + s.limiter.fail("ip:" + ip) + s.log.Warn("metrics: wrong token", "ip", ip) + return http.StatusUnauthorized, "wrong token for /metrics" + } + return 0, "" + } + if pr := peerOf(r.Context()); pr != nil { + if trustedUID(pr.UID) { + return 0, "" + } + return http.StatusForbidden, "on the local socket, only root and jarld's own user may read /metrics without metrics.token" + } + if a, err := netip.ParseAddr(ip); err == nil { + a = a.Unmap().WithZone("") + for _, p := range m.Allow { + if p.Contains(a) { + return 0, "" + } + } + } + if m.Token != "" { + return http.StatusUnauthorized, "send metrics.token as 'Authorization: Bearer TOKEN', or add " + ip + " to metrics.allow in jarl.yaml" + } + return http.StatusForbidden, ip + " may not read /metrics: add it to metrics.allow in jarl.yaml (or set metrics.token)" +} + +// serveMetricsListeners serves /metrics alone on metrics.listen. +func (s *Server) serveMetricsListeners(errc chan error) error { + if !s.cfg.Metrics.Enabled || len(s.cfg.Metrics.Listen) == 0 { + return nil + } + mux := http.NewServeMux() + mux.HandleFunc("GET /metrics", s.metrics) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "this address serves only /metrics", http.StatusNotFound) + }) + h := s.recoverer(s.logRequests(mux)) + for _, addr := range s.cfg.Metrics.Listen { + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("metrics.listen %s: %w", addr, err) + } + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: time.Minute, IdleTimeout: 2 * time.Minute, + ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelWarn)} + s.mu.Lock() + s.srvs = append(s.srvs, srv) + s.mu.Unlock() + s.log.Info("serving metrics", "url", "http://"+ln.Addr().String()+"/metrics") + go func() { + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + errc <- err + } + }() + } + return nil +} + +// stateMetrics renders Jarl's own state (cached for metricsStateTTL). +func (s *Server) stateMetrics(ctx context.Context) []byte { + c := &s.metricsCache + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + if c.body != nil && now.Sub(c.at) < metricsStateTTL { + return c.body + } + pw := &promWriter{} + s.collectState(ctx, pw, now) + var buf bytes.Buffer + pw.writeTo(&buf) + c.body, c.at = buf.Bytes(), now + return c.body +} + +func (s *Server) collectState(ctx context.Context, pw *promWriter, now time.Time) { + pw.gauge("jarl_build_info", "Jarl's version (always 1).", 1, "version", s.cfg.Version) + pw.gauge("jarl_start_time_seconds", "When jarld started (Unix time).", unix(s.started)) + ok := 1.0 + defer func() { + pw.gauge("jarl_metrics_state_ok", "1 when Jarl's state could be read for this scrape.", ok) + }() + + st, err := s.eng.Status(ctx) + if err != nil { + s.log.Warn("metrics: reading the status failed", "err", err) + ok = 0 + return + } + pw.gauge("jarl_adopted", "1 when named loads Jarl's configuration.", b2f(st.Adopted)) + if st.Named != nil || st.NamedError != "" { + pw.gauge("jarl_named_up", "1 when named answers rndc status.", b2f(st.Named != nil && st.Named.Running)) + } + if n := st.Named; n != nil { + if v := fractions(n.RecursiveClients); len(v) == 3 { + pw.gauge("jarl_named_recursive_clients", "Recursive clients named is serving now (rndc status).", v[0]) + pw.gauge("jarl_named_recursive_clients_soft_limit", "Recursive clients at which named starts dropping the oldest queries.", v[1]) + pw.gauge("jarl_named_recursive_clients_limit", "Most recursive clients named serves at once (recursive-clients).", v[2]) + } + if v := fractions(n.TCPClients); len(v) == 2 { + pw.gauge("jarl_named_tcp_clients", "TCP clients named is serving now (rndc status).", v[0]) + pw.gauge("jarl_named_tcp_clients_limit", "Most TCP clients named serves at once (tcp-clients).", v[1]) + } + } + pw.gauge("jarl_open_changesets", "Changesets with staged changes that are not applied yet.", float64(st.OpenChangesets)) + if st.LastApplied != nil && st.LastApplied.AppliedAt != nil { + pw.gauge("jarl_last_apply_timestamp_seconds", "When the last changeset was applied (Unix time).", unix(*st.LastApplied.AppliedAt)) + } + sev := map[validate.Severity]int{validate.Error: 0, validate.Warning: 0, validate.Info: 0} + for _, p := range st.Problems { + sev[p.Severity]++ + } + for _, k := range []validate.Severity{validate.Error, validate.Warning, validate.Info} { + pw.gauge("jarl_problems", "Validation problems in the current configuration, as on the dashboard.", float64(sev[k]), "severity", string(k)) + } + pw.gauge("jarl_drifted_files", "Files Jarl manages that were changed or removed outside Jarl.", float64(len(st.Drift))) + for _, k := range sortedKeys(st.Counts) { + pw.gauge("jarl_objects", "Objects in Jarl's configuration.", float64(st.Counts[k]), "kind", k) + } + + if s.cfg.BackupDir != "" { + bs := s.backupStatus(ctx, now) + if bs.LastAt != nil { + pw.gauge("jarl_backup_last_success_timestamp_seconds", "When the newest backup was made by 'jarld backup' (Unix time).", unix(*bs.LastAt)) + pw.gauge("jarl_backup_last_size_bytes", "Size of the newest backup.", float64(bs.LastSize)) + } + if bs.LastErrorAt != nil { + pw.gauge("jarl_backup_last_failure_timestamp_seconds", "When a backup last failed, if after the newest backup (Unix time).", unix(*bs.LastErrorAt)) + } + if s.cfg.BackupWarnAfter > 0 { + pw.gauge("jarl_backup_stale", "1 when the newest backup is older than backup.warn_after.", b2f(bs.Stale)) + } + } + + state, err := s.store.LoadState(ctx) + if err != nil { + ok = 0 + return + } + if len(state.Blocklists) > 0 || len(state.BlockRules) > 0 { + pw.gauge("jarl_blocking_paused", "1 while blocking is paused.", b2f(state.Settings.BlockingPaused())) + if t := state.Settings.BlockingPausedUntil; t != nil { + pw.gauge("jarl_blocking_paused_until_timestamp_seconds", "When paused blocking resumes (Unix time).", unix(*t)) + } + } + statuses, _ := s.store.BlocklistStatuses(ctx) + for _, b := range state.SortedBlocklists() { + pw.gauge("jarl_blocklist_enabled", "1 for blocklists that are switched on.", b2f(!b.Disabled), "list", b.Name) + bs := statuses[b.ID] + if bs == nil { + continue + } + pw.gauge("jarl_blocklist_entries", "Domains in the blocklist's last good download.", float64(bs.Entries), "list", b.Name) + pw.gauge("jarl_blocklist_failing", "1 when the last download or check of an enabled blocklist failed.", b2f(bs.Error != "" && !b.Disabled), "list", b.Name) + if bs.CheckedAt != nil { + pw.gauge("jarl_blocklist_last_check_timestamp_seconds", "When the blocklist was last checked for changes (Unix time).", unix(*bs.CheckedAt), "list", b.Name) + } + if bs.UpdatedAt != nil { + pw.gauge("jarl_blocklist_last_change_timestamp_seconds", "When the blocklist's content last changed (Unix time).", unix(*bs.UpdatedAt), "list", b.Name) + } + } + + zones := make([]*model.Zone, 0, len(state.Zones)) + for _, z := range state.Zones { + if z.DNSSECPolicy != "" && !z.Disabled { + zones = append(zones, z) + } + } + slices.SortFunc(zones, func(a, b *model.Zone) int { + return strings.Compare(a.Name+" "+state.ViewName(a), b.Name+" "+state.ViewName(b)) + }) + for _, z := range zones { + keys, err := dnssec.Keys(render.KeyDir(state, z, st.Paths), z.Name) + if err != nil { + continue + } + ph := dnssec.Assess(z.DNSSECPolicy, keys) + pw.gauge("jarl_dnssec_phase", "The DNSSEC phase of each signed zone (always 1; see docs/dnssec.md). ready: add the DS at the parent; removing_ds: remove it.", 1, + "zone", engine.Display(z.Name), "view", state.ViewName(z), "phase", ph.Code) + } + + if tokens, err := s.store.ListTokens(ctx, 0); err == nil { + for _, t := range tokens { + if t.ExpiresAt != nil && t.RevokedAt == nil { + pw.gauge("jarl_token_expiry_timestamp_seconds", "When each API token with an expiry date expires (Unix time).", unix(*t.ExpiresAt), + "id", strconv.FormatInt(t.ID, 10), "user", t.Username, "token", t.Name) + } + } + } +} + +// counterMetrics maps the statistics counters (docs/stats.md) to metrics: +// a key "PREFIX:VALUE" becomes metric{label="VALUE"}, a key without a +// colon a metric without labels. +var counterMetrics = map[string]struct{ name, label, help string }{ + "queries": {"jarl_queries_total", "", "Queries named received (statistics channel)."}, + "recursion": {"jarl_recursion_queries_total", "", "Queries that caused recursion."}, + "dropped": {"jarl_dropped_queries_total", "", "Queries named dropped."}, + "duplicate": {"jarl_duplicate_queries_total", "", "Duplicate queries named ignored."}, + "rcode": {"jarl_responses_by_rcode_total", "rcode", "Responses by response code."}, + "qtype": {"jarl_queries_by_type_total", "type", "Queries by record type."}, + "answer": {"jarl_answers_total", "source", "Answers that were authoritative or not (recursive)."}, + "transport": {"jarl_queries_by_transport_total", "transport", "Queries over UDP and TCP."}, + "ratelimit": {"jarl_ratelimit_responses_total", "action", "Responses rate limiting dropped or slipped (truncated)."}, + "rtt": {"jarl_upstream_responses_by_rtt_total", "rtt", "Responses from other servers during recursion, by round-trip time."}, + "dnssec": {"jarl_dnssec_validations_total", "result", "DNSSEC validations of recursive answers by result."}, + "cache": {"jarl_cache_lookups_total", "result", "Cache lookups that hit or missed."}, + "responses": {"jarl_logged_responses_total", "", "Responses read from named's response log (without stats.exclude_clients)."}, + "view": {"jarl_responses_by_view_total", "view", "Logged responses by view."}, + "zone": {"jarl_responses_by_zone_total", "zone", "Logged responses by local zone (\"-\": names in no local zone)."}, + "blocked": {"jarl_blocked_queries_total", "", "Queries blocking answered."}, + "blocklist": {"jarl_blocked_by_list_total", "list", "Blocked queries by blocklist (\"rules\": Jarl's own rules)."}, +} + +// statsMetrics adds the query statistics: counters since jarld started, +// and named's process and server gauges. +func statsMetrics(pw *promWriter, totals map[string]int64, srv *stats.Server, st stats.Status) { + if _, ok := totals["queries"]; !ok { + totals["queries"] = 0 + } + for _, k := range sortedKeys(totals) { + prefix, value, labeled := strings.Cut(k, ":") + m, ok := counterMetrics[prefix] + if !ok || labeled != (m.label != "") { + continue + } + if labeled { + pw.counter(m.name, m.help, float64(totals[k]), m.label, value) + } else { + pw.counter(m.name, m.help, float64(totals[k])) + } + } + pw.gauge("jarl_stats_poll_ok", "1 when the last poll of named's statistics channel worked.", b2f(st.PollError == "")) + if st.LastPoll != nil { + pw.gauge("jarl_stats_last_poll_timestamp_seconds", "When named's statistics channel was last read (Unix time).", unix(*st.LastPoll)) + } + if st.QueryLog { + pw.counter("jarl_stats_log_lines_total", "Lines read from named's response log.", float64(st.LinesRead)) + if st.LastLine != nil { + pw.gauge("jarl_stats_last_log_line_timestamp_seconds", "Time of the last response log line read (Unix time).", unix(*st.LastLine)) + } + } + pw.gauge("jarl_stats_warnings", "Warnings about statistics collection, as on the Stats page.", float64(len(st.Warnings))) + if srv == nil { + return + } + if t, err := time.Parse(time.RFC3339Nano, srv.BootTime); err == nil { + pw.gauge("jarl_named_start_time_seconds", "When named started (Unix time).", unix(t)) + } + if t, err := time.Parse(time.RFC3339Nano, srv.ConfigTime); err == nil { + pw.gauge("jarl_named_config_time_seconds", "When named last loaded its configuration (Unix time).", unix(t)) + } + if srv.PID != 0 { + pw.gauge("jarl_named_resident_memory_bytes", "named's resident memory (/proc).", float64(srv.RSS)) + pw.gauge("jarl_named_threads", "named's threads (/proc).", float64(srv.Threads)) + pw.counter("jarl_named_cpu_seconds_total", "CPU time named used since it started (/proc).", srv.CPUSeconds) + } + if srv.Version != "" { + pw.gauge("jarl_named_memory_in_use_bytes", "Memory named counts as in use (statistics channel).", float64(srv.MemInUse)) + pw.gauge("jarl_named_cache_rrsets", "RRsets in named's caches, all views.", float64(srv.CacheRRsets)) + pw.gauge("jarl_named_cache_memory_bytes", "Memory named's caches use, all views.", float64(srv.CacheMem)) + pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsUDP), "proto", "udp") + pw.gauge("jarl_named_sockets", "Open sockets.", float64(srv.SocketsTCP), "proto", "tcp") + pw.gauge("jarl_named_recursive_clients_highwater", "Most recursive clients at once since named started.", float64(srv.RecursionHighwater)) + pw.gauge("jarl_named_tcp_clients_highwater", "Most TCP clients at once since named started.", float64(srv.TCPHighwater)) + } +} + +// promWriter collects metrics and writes them in Prometheus's text format, +// each metric's samples together under one HELP and TYPE. +type promWriter struct { + order []string + fams map[string]*promFamily +} + +type promFamily struct { + typ, help string + lines []string +} + +func (w *promWriter) gauge(name, help string, v float64, labels ...string) { + w.add(name, "gauge", help, v, labels) +} + +func (w *promWriter) counter(name, help string, v float64, labels ...string) { + w.add(name, "counter", help, v, labels) +} + +func (w *promWriter) add(name, typ, help string, v float64, labels []string) { + if w.fams == nil { + w.fams = map[string]*promFamily{} + } + f := w.fams[name] + if f == nil { + f = &promFamily{typ: typ, help: help} + w.fams[name] = f + w.order = append(w.order, name) + } + var b strings.Builder + b.WriteString(name) + if len(labels) > 0 { + b.WriteByte('{') + for i := 0; i+1 < len(labels); i += 2 { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(labels[i]) + b.WriteString(`="`) + b.WriteString(labelEscaper.Replace(labels[i+1])) + b.WriteByte('"') + } + b.WriteByte('}') + } + b.WriteByte(' ') + b.WriteString(formatFloat(v)) + f.lines = append(f.lines, b.String()) +} + +func (w *promWriter) writeTo(buf *bytes.Buffer) { + for _, name := range w.order { + f := w.fams[name] + fmt.Fprintf(buf, "# HELP %s %s\n# TYPE %s %s\n", name, helpEscaper.Replace(f.help), name, f.typ) + for _, l := range f.lines { + buf.WriteString(l) + buf.WriteByte('\n') + } + } +} + +var ( + labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`) + helpEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`) +) + +func formatFloat(v float64) string { + switch { + case math.IsInf(v, 1): + return "+Inf" + case math.IsInf(v, -1): + return "-Inf" + case math.IsNaN(v): + return "NaN" + } + return strconv.FormatFloat(v, 'f', -1, 64) +} + +func unix(t time.Time) float64 { return float64(t.UnixMilli()) / 1000 } + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// fractions parses rndc status values such as "0/900/1000". +func fractions(s string) []float64 { + if s == "" { + return nil + } + var out []float64 + for _, p := range strings.Split(s, "/") { + v, err := strconv.ParseFloat(strings.TrimSpace(p), 64) + if err != nil { + return nil + } + out = append(out, v) + } + return out +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/server/metrics_test.go b/internal/server/metrics_test.go new file mode 100644 index 0000000..09e4b90 --- /dev/null +++ b/internal/server/metrics_test.go @@ -0,0 +1,206 @@ +package server + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/netip" + "regexp" + "strings" + "testing" + "time" + + "jarl/internal/stats" +) + +// scrape GETs /metrics with an optional bearer token. +func (e *apiEnv) scrape(token string) (string, int) { + e.t.Helper() + r, _ := http.NewRequest("GET", e.ts.URL+"/metrics", nil) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(r) + if err != nil { + e.t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return string(b), resp.StatusCode +} + +var ( + promSampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*"(,[a-zA-Z_][a-zA-Z0-9_]*="(?:[^"\\]|\\.)*")*\})? -?[0-9.eE+-]+$`) + promTypeRe = regexp.MustCompile(`^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (gauge|counter)$`) +) + +// checkExposition checks Prometheus's text format: valid lines, and each +// metric's samples together under a single TYPE. +func checkExposition(t *testing.T, body string) { + t.Helper() + typed := map[string]bool{} + cur := "" + for _, l := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") { + switch { + case strings.HasPrefix(l, "# HELP "): + case strings.HasPrefix(l, "# TYPE "): + m := promTypeRe.FindStringSubmatch(l) + if m == nil { + t.Errorf("bad TYPE line %q", l) + continue + } + if typed[m[1]] { + t.Errorf("metric %s appears twice", m[1]) + } + typed[m[1]], cur = true, m[1] + default: + if !promSampleRe.MatchString(l) { + t.Errorf("bad sample line %q", l) + } + if name, _, _ := strings.Cut(strings.SplitN(l, " ", 2)[0], "{"); name != cur { + t.Errorf("sample %q is not under its TYPE (%s)", l, cur) + } + } + } +} + +func TestMetricsAccess(t *testing.T) { + loopback := []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("::1/128")} + e := newAPIEnv(t) + if _, status := e.scrape(""); status != 404 { + t.Errorf("disabled = %d, want 404", status) + } + + e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: loopback}}) + body, status := e.scrape("") + if status != 200 { + t.Fatalf("allowed scrape = %d: %s", status, body) + } + checkExposition(t, body) + for _, want := range []string{`jarl_build_info{version="test"} 1`, `jarl_problems{severity="error"} 0`, `jarl_objects{kind="zones"} 0`, "jarl_open_changesets 0", "jarl_metrics_state_ok 1"} { + if !strings.Contains(body, want+"\n") { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if strings.Contains(body, "jarl_queries_total") { + t.Error("query counters without statistics") + } + // No token is configured: one that is sent is refused. + if _, status := e.scrape("whatever-token-123"); status != 401 { + t.Errorf("token without metrics.token = %d, want 401", status) + } + + e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}}}) + if body, status := e.scrape(""); status != 403 || !strings.Contains(body, "metrics.allow") { + t.Errorf("not allowed = %d %s", status, body) + } + + const token = "0123456789abcdef-metrics" + e = newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Token: token}}) + if _, status := e.scrape(""); status != 401 { + t.Errorf("no token = %d, want 401", status) + } + if _, status := e.scrape("wrong-token-0123456789"); status != 401 { + t.Errorf("wrong token = %d, want 401", status) + } + if _, status := e.scrape(token); status != 200 { + t.Errorf("right token = %d, want 200", status) + } + // The API's tokens don't count for /metrics. + if _, status := e.scrape(e.token); status != 401 { + t.Errorf("API token = %d, want 401", status) + } +} + +func TestMetricsState(t *testing.T) { + e := newAPIEnvConfig(t, Config{Version: "test", Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}}) + e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201) + e.do("POST", "/blocking/lists", e.token, map[string]any{"name": "ads", "url": "https://lists.example/ads.txt"}, 201) + e.srv.metricsCache.body = nil // skip the cache + body, _ := e.scrape("") + checkExposition(t, body) + for _, want := range []string{`jarl_objects{kind="zones"} 1`, `jarl_blocklist_enabled{list="ads"} 1`, "jarl_blocking_paused 0"} { + if !strings.Contains(body, want+"\n") { + t.Errorf("missing %q in:\n%s", want, body) + } + } + // Cached: a change shows after metricsStateTTL. + e.do("POST", "/zones", e.token, map[string]any{"name": "example.org", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}}}, 201) + if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 1`+"\n") { + t.Error("state was not cached") + } + e.srv.metricsCache.at = time.Now().Add(-metricsStateTTL) + if body, _ := e.scrape(""); !strings.Contains(body, `jarl_objects{kind="zones"} 2`+"\n") { + t.Errorf("state not refreshed:\n%s", body) + } +} + +func TestMetricsStats(t *testing.T) { + ss, err := stats.OpenStore(":memory:") + if err != nil { + t.Fatal(err) + } + defer ss.Close() + opts := stats.Options{QueryLog: true, LogFile: "/nonexistent", StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour} + c := stats.NewCollector(ss, opts, func() map[string][]string { return map[string][]string{"internal": {"example.com"}} }, slog.New(slog.NewTextHandler(io.Discard, nil))) + ts := time.Now().Format("02-Jan-2006 15:04:05.000") + c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5353 (www.example.com): view internal: response: www.example.com IN A NOERROR 1 0 1 +E(0)K (10.10.0.2)") + c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: rpz QNAME NXDOMAIN rewrite ads.example/A/IN via ads.example.hagezi.blocklist.jarl.invalid") + c.HandleLine(ts + " client @0x7f0000000000 10.0.0.9#5354 (ads.example): view internal: response: ads.example IN A NXDOMAIN 1 0 1 +E(0)K (10.10.0.2)") + + e := newAPIEnvConfig(t, Config{Version: "test", Stats: c, Metrics: Metrics{Enabled: true, Allow: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")}}}) + body, status := e.scrape("") + if status != 200 { + t.Fatalf("scrape = %d", status) + } + checkExposition(t, body) + for _, want := range []string{ + "jarl_queries_total 0", "jarl_logged_responses_total 2", `jarl_responses_by_view_total{view="internal"} 2`, + `jarl_responses_by_zone_total{zone="example.com"} 1`, `jarl_responses_by_zone_total{zone="-"} 1`, + "jarl_blocked_queries_total 1", `jarl_blocked_by_list_total{list="hagezi"} 1`, "jarl_stats_log_lines_total 2", + } { + if !strings.Contains(body, want+"\n") { + t.Errorf("missing %q in:\n%s", want, body) + } + } +} + +func TestStatsMetricsMapping(t *testing.T) { + pw := &promWriter{} + totals := map[string]int64{"queries": 10, "rcode:NOERROR": 8, "rcode:NXDOMAIN": 2, "qtype:A": 10, "ratelimit:slipped": 3, + "cache:hits": 4, "rtt:<10ms": 5, "unknown:x": 1, "rcode": 7} + srv := &stats.Server{BootTime: "2026-10-07T20:00:00.000Z", Version: "9.20.4", PID: 42, RSS: 1 << 20, CPUSeconds: 1.5, SocketsUDP: 4} + statsMetrics(pw, totals, srv, stats.Status{}) + var buf bytes.Buffer + pw.writeTo(&buf) + out := buf.String() + checkExposition(t, out) + for _, want := range []string{ + "jarl_queries_total 10", `jarl_responses_by_rcode_total{rcode="NXDOMAIN"} 2`, `jarl_ratelimit_responses_total{action="slipped"} 3`, + `jarl_upstream_responses_by_rtt_total{rtt="<10ms"} 5`, `jarl_cache_lookups_total{result="hits"} 4`, + "jarl_named_start_time_seconds 1791403200", "jarl_named_cpu_seconds_total 1.5", `jarl_named_sockets{proto="udp"} 4`, + "jarl_named_resident_memory_bytes 1048576", + } { + if !strings.Contains(out, want+"\n") { + t.Errorf("missing %q in:\n%s", want, out) + } + } + if strings.Contains(out, "unknown") || strings.Contains(out, "jarl_responses_by_rcode_total 7") { + t.Errorf("unknown keys exported:\n%s", out) + } +} + +func TestPromWriterEscaping(t *testing.T) { + pw := &promWriter{} + pw.gauge("m", "a \\ help\nline", 1, "l", "a\"b\\c\nd") + var buf bytes.Buffer + pw.writeTo(&buf) + want := "# HELP m a \\\\ help\\nline\n# TYPE m gauge\nm{l=\"a\\\"b\\\\c\\nd\"} 1\n" + if buf.String() != want { + t.Errorf("got %q, want %q", buf.String(), want) + } + if got := formatFloat(1759867200.123); got != "1759867200.123" { + t.Errorf("formatFloat = %s", got) + } +} diff --git a/internal/server/routes.go b/internal/server/routes.go index c2b5c76..15d1faa 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -22,6 +22,7 @@ func (s *Server) routes() { m.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": s.cfg.Version}) }) + m.HandleFunc("GET /metrics", s.metrics) // Authentication m.HandleFunc("POST "+p+"/auth/login", s.public(s.login)) m.HandleFunc("POST "+p+"/auth/login/totp", s.public(s.loginTOTP)) diff --git a/internal/server/server.go b/internal/server/server.go index e42007f..772b386 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -58,6 +58,8 @@ type Config struct { // name authenticator apps show (h_totp.go). TOTPRequire string TOTPIssuer string + // Metrics configures /metrics (metrics.go). + Metrics Metrics } // Server is the HTTP server. @@ -80,6 +82,10 @@ type Server struct { // nsChecker, when set, replaces the default name server checker // (tests point it at local DNS servers). nsChecker *nscheck.Checker + // started is when the server was created; metricsCache holds the Jarl + // state part of /metrics. + started time.Time + metricsCache metricsCache mu sync.Mutex srvs []*http.Server @@ -108,7 +114,7 @@ func New(cfg Config, eng *engine.Engine, log *slog.Logger, web fs.FS) *Server { if log == nil { log = slog.Default() } - s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web} + s := &Server{cfg: cfg, eng: eng, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(), web: web, started: time.Now()} s.expiryWarn.every = 24 * time.Hour if cfg.Stats != nil { s.statsQ = stats.NewQuerier(cfg.Stats) @@ -200,6 +206,10 @@ func (s *Server) Serve(ctx context.Context, ready func()) error { } } } + if err := s.serveMetricsListeners(errc); err != nil { + s.shutdown() + return err + } s.serveSocket(errc) s.logHTTPReq(tlsCfg != nil) if ready != nil { diff --git a/internal/stats/collector.go b/internal/stats/collector.go index f908c10..45d4b8f 100644 --- a/internal/stats/collector.go +++ b/internal/stats/collector.go @@ -114,6 +114,9 @@ type Collector struct { // statistics (server.go). proc *namedProc server *Server + // totals are the counters added up since jarld started (gauges left + // out), for Prometheus (Totals). + totals map[string]int64 } // maxKeys bounds the per-hour maps; rarely seen keys are dropped first. @@ -125,7 +128,7 @@ func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger) opts.Location = time.Local } c := &Collector{opts: opts, store: store, zones: zones, log: log, poller: NewPoller(opts.StatisticsChannel), now: time.Now, - minutes: map[int64]map[string]int64{}, hours: map[int64]map[string]map[string]int64{}, late: map[int64]map[string]map[string]int64{}} + minutes: map[int64]map[string]int64{}, totals: map[string]int64{}, hours: map[int64]map[string]map[string]int64{}, late: map[int64]map[string]map[string]int64{}} if opts.QueryLog { c.tail = &Tailer{Path: opts.LogFile} } @@ -286,6 +289,23 @@ func (c *Collector) count(minute int64, key string, n int64) { c.minutes[minute] = m } m[key] += n + if !strings.HasPrefix(key, "g:") && !strings.HasPrefix(key, "gn:") { + c.totals[key] += n + } +} + +// Totals returns every counter added up since the collector started (the +// keys of docs/stats.md, without gauges). They only grow, so they suit +// Prometheus counters; named's counters move once a minute, when they are +// polled. +func (c *Collector) Totals() map[string]int64 { + c.mu.Lock() + defer c.mu.Unlock() + out := make(map[string]int64, len(c.totals)) + for k, v := range c.totals { + out[k] = v + } + return out } // readLog reads new response log lines. diff --git a/internal/stats/server.go b/internal/stats/server.go index 810f01c..6dbe8c5 100644 --- a/internal/stats/server.go +++ b/internal/stats/server.go @@ -14,7 +14,8 @@ type Server struct { PID int `json:"pid,omitempty"` RSS int64 `json:"rss,omitempty"` // resident memory, bytes Threads int64 `json:"threads,omitempty"` - CPUPercent float64 `json:"cpu_percent"` // of one CPU, over the last minute + CPUPercent float64 `json:"cpu_percent"` // of one CPU, over the last minute + CPUSeconds float64 `json:"cpu_seconds,omitempty"` // user + system time since named started // From the statistics channel. MemInUse int64 `json:"mem_in_use"` // named's own count of the memory it uses CacheRRsets int64 `json:"cache_rrsets"` @@ -63,6 +64,7 @@ func (c *Collector) sampleServer(minute int64, now time.Time, snap *Snapshot) { c.proc = nil } else { srv.PID, srv.RSS, srv.Threads = p.PID, p.RSS, p.Threads + srv.CPUSeconds = float64(p.CPU) / clockTicks if prev := c.proc; prev.At.Before(now) && !prev.At.IsZero() && p.CPU >= prev.CPU { // Tenths of a percent of one CPU: ticks / clockTicks is CPU seconds. permille := (p.CPU - prev.CPU) * 1000 / clockTicks / max(int64(now.Sub(prev.At)/time.Second), 1) diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 7642bfb..14de49e 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -13,7 +13,7 @@ set -u ROOT=$(cd "$(dirname "$0")/.." && pwd) WORK=${WORK:-$(mktemp -d /tmp/jarl-e2e.XXXXXX)} E=$WORK -DNS=${DNS_PORT:-25353}; CTL=${CTL_PORT:-25953}; HTTP=${HTTP_PORT:-28053}; LISTS=${LISTS_PORT:-28054} +DNS=${DNS_PORT:-25353}; CTL=${CTL_PORT:-25953}; HTTP=${HTTP_PORT:-28053}; LISTS=${LISTS_PORT:-28054}; METRICS=${METRICS_PORT:-28055} if [ -n "${JARL_BIND_ROOT:-}" ]; then B=$JARL_BIND_ROOT export LD_LIBRARY_PATH=$(echo $B/usr/lib/*-linux-gnu | cut -d' ' -f1) @@ -108,6 +108,10 @@ bind: rndc_args: ["-k", "$E/legacy/rndc.key", "-s", "127.0.0.1", "-p", "$CTL"] backup: dir: $E/backups +metrics: + enabled: true + listen: ["127.0.0.1:$METRICS"] + token: e2e-metrics-token-0123456789 EOF cd $E echo "secret-pass-123" | $JARLD -c jarl.yaml init --password-stdin --token --nameservers ns1.adbraun.test. --contact admin@adbraun.test > init.out 2>&1 || { cat init.out; exit 1; } @@ -561,6 +565,22 @@ run "block delete (allow rule)" 0 $J block delete ok.tracker.adbraun.test run "no blocking left in named.conf" 1 grep -q 'response-policy' $E/etc-jarl/named.conf kill $LISTS_PID 2>/dev/null; LISTS_PID= +# Prometheus metrics (docs/metrics.md) +run "metrics on the main listener" 0 curl -sf http://127.0.0.1:$HTTP/metrics +contains "^jarl_named_up 1$" +contains "^jarl_adopted 1$" +contains '^jarl_problems{severity="error"} 0$' +contains '^jarl_objects{kind="zones"} ' +run "metrics on metrics.listen" 0 curl -sf http://127.0.0.1:$METRICS/metrics +contains "^jarl_build_info{version=" +run "metrics.listen serves nothing else" 0 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:$METRICS/api/v1/zones +contains "^404$" +run "metrics with the token" 0 curl -sf -H "Authorization: Bearer e2e-metrics-token-0123456789" http://127.0.0.1:$METRICS/metrics +run "metrics with a wrong token" 0 curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer wrong-token-0123456789" http://127.0.0.1:$METRICS/metrics +contains "^401$" +run "metrics on the socket" 0 curl -sf --unix-socket $SOCK http://jarl/metrics +contains "^jarl_open_changesets " + run "status at the end" 0 $J status contains "Drift:.*none"