Compare commits

..
2 Commits
Author SHA1 Message Date
adamandClaude Opus 5.5 01ec37b9b2 Explore the statistics: a query log to drill down by client, name, view and more; storage on the dashboard
jarld keeps every answer for a while (stats.retention.queries, default 7 days) in the statistics database: time, client, view, name, type, result and what blocked it. Every client, name, view, record type, result and blocklist on the Statistics page leads to an explore page for it: answers over time, the top values of every other dimension (each a link that narrows further) and the latest answers. GET /stats/explore.

The dashboard shows the size of Jarl's databases (and the query log's rows) and, per disk Jarl uses, how full it is and the databases' share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 18:15:36 +01:00
adamandClaude Opus 5.5 fef916b7ab Encrypted DNS: several host names, views per listen address, what named uses a certificate for
The setup takes several host names (one certificate for all; the health check says when the served certificate misses one), stored in the settings. Each listen address gets its own views for the host names' A/AAAA records (--address ADDR=VIEW,..., a view checkbox per address in the form), always local to the view, with a warning when a public view would get a private address; a view no longer given loses the address, other records of the names stay, and the form starts from where the records are now. Certificates (page and jarl cert show) say what named uses each for: DNS over TLS or HTTPS, on which addresses and ports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 17:49:49 +01:00
38 changed files with 1509 additions and 149 deletions

No files matched your search

+22
View File
@@ -48,9 +48,29 @@ All notable changes to Jarl are documented here. The format follows
manageable: the public addresses fold away, show 25 at a time, and
**Dismiss these N** dismisses everything a search found in one change
(`--dismiss` takes several networks).
- Encrypted DNS: **several host names** (one certificate for all; the
health check says when the served certificate misses one), and **each
listen address gets its own views** for the host names' records
(`--address 10.0.10.5=internal`, a view checkbox per address in the
form), local to the view, with a warning when a public view would get
a private address. A view no longer ticked loses the address. The form
starts from where the records are now. The Certificates page and `jarl
cert show` say what named uses a certificate for (DNS over TLS or HTTPS,
on which addresses and ports).
- Comparing one name across views: `jarl zone diff adbraun.lol nas [A]`,
and a **Compare** button on each record.
- **Exploring the statistics**: jarld keeps every answer for a while
(`stats.retention.queries`, default 7 days), and every client, name,
view, record type, result and blocklist on the Statistics page leads to
an explore page for it: answers over time, the top clients, names,
views, types, results and blocklists among them (each a link that
narrows it further), and the latest answers. "NXDOMAIN in external" →
the clients asking → everything one of them asks. `GET /stats/explore`.
- **Storage** on the dashboard: the size of Jarl's databases (and how many
answers the query log holds), and per disk Jarl uses a bar of how full it
is and how much of it the databases take.
### Changed
- **Compare views** and `jarl zone diff` are rebuilt around the view
@@ -74,6 +94,8 @@ All notable changes to Jarl are documented here. The format follows
starts; a 0.5 jarld can't open it afterwards, so take a backup first (`sudo
jarld backup`). Nothing changes until you turn `auto_ptr` on for a
zone.
- The statistics database gets a new version too (the query log); 0.5
can't open it afterwards either. It starts empty and fills from then on.
## [0.5.2] - 2026-10-12
+4
View File
@@ -45,6 +45,7 @@ type certInfo struct {
NextAttempt *time.Time `json:"next_attempt,omitempty"`
IssuingSince *time.Time `json:"issuing_since,omitempty"`
Serving bool `json:"serving,omitempty"`
NamedUse []string `json:"named_use,omitempty"`
Files *struct {
Chain string `json:"chain"`
Key string `json:"key"`
@@ -173,6 +174,9 @@ func printCert(w io.Writer, ci certInfo) {
}
}
fmt.Fprintf(w, "Uses:\t%s\n", dash(strings.Join(uses, ", ")))
for _, u := range ci.NamedUse {
fmt.Fprintf(w, "named:\t%s\n", u)
}
if ci.Comment != "" {
fmt.Fprintf(w, "Comment:\t%s\n", ci.Comment)
}
+88 -32
View File
@@ -17,15 +17,17 @@ type encEndpoint struct {
}
type encStatus struct {
Enabled bool `json:"enabled"`
Certificate string `json:"certificate"`
Hostname string `json:"hostname"`
DoTHost string `json:"dot_host"`
DoHURL string `json:"doh_url"`
DoT []encEndpoint `json:"dot"`
DoH []encEndpoint `json:"doh"`
Other bool `json:"other"`
Health *encHealth `json:"health"`
Enabled bool `json:"enabled"`
Certificate string `json:"certificate"`
Hostname string `json:"hostname"`
Hostnames []string `json:"hostnames"`
AddrViews map[string][]string `json:"address_views"`
DoTHost string `json:"dot_host"`
DoHURL string `json:"doh_url"`
DoT []encEndpoint `json:"dot"`
DoH []encEndpoint `json:"doh"`
Other bool `json:"other"`
Health *encHealth `json:"health"`
Addresses []struct {
Address string `json:"address"`
Private bool `json:"private"`
@@ -87,7 +89,14 @@ views.`,
if !s.Enabled {
fmt.Fprintln(w, "Encrypted DNS:\toff (jarl encrypted-dns setup --hostname NAME --address ADDR)")
} else {
fmt.Fprintf(w, "Host names:\t%s\n", strings.Join(s.Hostnames, ", "))
fmt.Fprintf(w, "Certificate:\t%s\n", s.Certificate)
for _, e := range append(append([]encEndpoint{}, s.DoT...), s.DoH...) {
if vs, ok := s.AddrViews[e.Address]; ok {
fmt.Fprintf(w, "Records:\t%s in %s\n", e.Address, strings.Join(vs, ", "))
delete(s.AddrViews, e.Address)
}
}
if len(s.DoT) > 0 {
fmt.Fprintf(w, "DNS over TLS:\t%s\n", endpoints(s.DoT))
}
@@ -119,12 +128,13 @@ views.`,
})
},
}
var host, certName string
var hostList []string
var certName string
var addrs, views []string
var noDoT, noDoH bool
var dohPort, dotPort int
setup := &cobra.Command{
Use: "setup --hostname NAME --address ADDR...",
Use: "setup --hostname NAME... --address ADDR[=VIEW,...]...",
Short: "Set up DNS over TLS and HTTPS (obtaining a certificate if needed)",
Long: `Sets up DoT (port 853) and DoH (port 443, or --doh-port) on the addresses
given, which should be this server's addresses on your own networks:
@@ -135,25 +145,57 @@ given, which should be this server's addresses on your own networks:
- named's tls statement and listen-on statements (and an explicit
listen-on { any; } when there was none, as the first listen-on would
otherwise stop named listening on port 53 everywhere);
- the host name's A and AAAA records, in the private views that have its
zone (or --view), local to the view when the zone is linked.
- the host names' A and AAAA records, local to each view: per address,
the views after = (--address 10.0.10.5=internal,lab), or the views of
--view for every address (default: the private views that have the
zone). A view no longer given loses the address.
It is one change, staged or applied like any other. Running it again
replaces the earlier setup.`,
Several --hostname: the certificate must be for all of them. It is one
change, staged or applied like any other. Running it again replaces the
earlier setup, ports included.`,
Example: ` jarl encrypted-dns setup --hostname dns.adbraun.lol --address 10.0.10.5=internal --address fdab:abab:abab:10::5=internal
jarl encrypted-dns setup --hostname dns.adbraun.lol --hostname doh.adbraun.lol --address 10.0.10.5 --view internal --dot-port 8853`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
c, err := newClient()
if err != nil {
return err
}
host = strings.ToLower(strings.TrimSuffix(host, "."))
if host == "" {
var hosts []string
for _, h := range hostList {
for _, x := range strings.Split(h, ",") {
if x = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(x), ".")); x != "" {
hosts = append(hosts, x)
}
}
}
if len(hosts) == 0 {
return errors.New("give --hostname, the name clients connect to (e.g. dns.adbraun.lol)")
}
host := hosts[0]
var s encStatus
if err := c.Get(ctx(), "/encrypted-dns", q("hostname", host), &s); err != nil {
if err := c.Get(ctx(), "/encrypted-dns", q("hostname", strings.Join(hosts, ",")), &s); err != nil {
return err
}
// ADDR=VIEW,VIEW gives an address its own views.
addrViews := map[string][]string{}
perAddress := false
for i, a := range addrs {
addr, vs, ok := strings.Cut(a, "=")
addrs[i] = strings.TrimSpace(addr)
if ok {
perAddress = true
addrViews[addrs[i]] = nil
for _, v := range strings.Split(vs, ",") {
if v = strings.TrimSpace(v); v != "" {
addrViews[addrs[i]] = append(addrViews[addrs[i]], v)
}
}
}
}
if perAddress && len(views) > 0 {
return errors.New("give the views per address (--address ADDR=VIEW,...) or for all of them (--view), not both")
}
if len(addrs) == 0 {
var list []string
for _, a := range s.Addresses {
@@ -163,20 +205,24 @@ replaces the earlier setup.`,
}
// The certificate.
if certName == "" {
certs:
for _, ci := range s.Certificates {
if coversName(append(ci.Names, ci.Domains...), host) {
certName = ci.Name
break
for _, h := range hosts {
if !coversName(append(ci.Names, ci.Domains...), h) {
continue certs
}
}
certName = ci.Name
break
}
}
if certName == "" {
certName = strings.SplitN(host, ".", 2)[0]
fmt.Printf("Creating certificate %s for %s\n", certName, host)
fmt.Printf("Creating certificate %s for %s\n", certName, strings.Join(hosts, ", "))
var r struct {
Certificate certInfo `json:"certificate"`
}
if err := c.Do(ctx(), "POST", "/certificates", nil, map[string]any{"name": certName, "domains": []string{host}, "uses": []string{"dns"}}, &r); err != nil {
if err := c.Do(ctx(), "POST", "/certificates", nil, map[string]any{"name": certName, "domains": hosts, "uses": []string{"dns"}}, &r); err != nil {
return err
}
if err := waitIssue(c, certName); err != nil {
@@ -204,7 +250,7 @@ replaces the earlier setup.`,
}
}
}
body := map[string]any{"hostname": host, "certificate": certName, "addresses": addrs, "dot": !noDoT, "doh": !noDoH}
body := map[string]any{"hostnames": hosts, "certificate": certName, "addresses": addrs, "dot": !noDoT, "doh": !noDoH}
if cmd.Flags().Changed("doh-port") {
body["doh_port"] = dohPort
}
@@ -234,14 +280,24 @@ replaces the earlier setup.`,
if chk.Error != "" && !flagQuiet {
fmt.Println("Note:", chk.Error)
}
if len(views) == 0 {
for _, v := range s.Views {
if !v.Public && v.HasZone {
views = append(views, v.Name)
switch {
case perAddress:
for _, a := range addrs {
if _, ok := addrViews[a]; !ok {
addrViews[a] = []string{} // no records for it
}
}
body["address_views"] = addrViews
default:
if len(views) == 0 {
for _, v := range s.Views {
if !v.Public && v.HasZone {
views = append(views, v.Name)
}
}
}
body["views"] = views
}
body["views"] = views
resp, err := c.Mutate(ctx(), "POST", "/encrypted-dns", nil, body)
if err != nil {
return err
@@ -250,14 +306,14 @@ replaces the earlier setup.`,
},
}
f := setup.Flags()
f.StringVar(&host, "hostname", "", "the name clients connect to (the certificate's)")
f.StringArrayVar(&hostList, "hostname", nil, "a name clients connect to (repeat, or comma separated, for several; the certificate must be for all)")
f.StringVar(&certName, "certificate", "", "the Jarl certificate to serve (default: one for the host name, obtained if there is none)")
f.StringArrayVar(&addrs, "address", nil, "this server's address to listen on (repeat for several)")
f.StringArrayVar(&addrs, "address", nil, "this server's address to listen on, with the views that get it: 10.0.10.5=internal,lab (repeat for several)")
f.BoolVar(&noDoT, "no-dot", false, "no DNS over TLS")
f.BoolVar(&noDoH, "no-doh", false, "no DNS over HTTPS")
f.IntVar(&dohPort, "doh-port", 443, "port for DNS over HTTPS")
f.IntVar(&dotPort, "dot-port", 853, "port for DNS over TLS (clients expect 853)")
f.StringArrayVar(&views, "view", nil, "views to add the host name's records to (default: the private views that have its zone)")
f.StringArrayVar(&views, "view", nil, "views where every address gets the host names' records (default: the private views that have the zone); per address: --address ADDR=VIEW")
var cAddrs []string
var cDoH int
+2 -1
View File
@@ -151,7 +151,8 @@ func statsOptions(cfg *config.Config) stats.Options {
sc := cfg.Stats
o := stats.Options{QueryLog: sc.QueryLog, LogFile: sc.LogFile, StatisticsChannel: sc.StatisticsChannel, TopN: sc.TopN,
RetainMinute: time.Duration(sc.Retention.Minute), RetainHour: time.Duration(sc.Retention.Hour), RetainDay: time.Duration(sc.Retention.Day),
Anonymize: sc.AnonymizeClients, MainConfig: cfg.BIND.MainConfig}
RetainQueries: time.Duration(sc.Retention.Queries),
Anonymize: sc.AnonymizeClients, MainConfig: cfg.BIND.MainConfig}
for _, p := range sc.ExcludeClients {
if pf, err := config.ParsePrefix(p); err == nil {
o.Exclude = append(o.Exclude, pf.Masked())
+1
View File
@@ -565,6 +565,7 @@ Needs `stats:read`. When statistics are disabled, every endpoint but
| GET | `/stats/timeline` | `since`, `step` (auto, 1m, 5m, 15m, 1h, 6h, 1d), `key` (repeatable or comma separated; `rcode:*` = every counter of a group; default `queries`) | `{step, times: [unix...], series: [{key, values}]}` |
| GET | `/stats/top` | `kind` (name, client, nxdomain, servfail, refused, blocked, blocked_client), `since`, `limit` (default 20) | `[{key, count}]`; client, refused and blocked_client entries add `name`, the host name of Jarl's record for the address (the summary's client list too) |
| GET | `/stats/heatmap` | `since` (default 28d), `key` (default queries) | `{key, from, avg[7][24], samples[7][24]}`, Monday first, local time, `-1` = no data |
| GET | `/stats/explore` | `?client=&name=&domain=&view=&type=&rcode=&blocked=yes\|no\|LIST&since=24h&views=` | The query log for a filter ([stats.md](stats.md#exploring)): `total`, `blocked`, `series` (per `step` seconds: `t`, `count`, `blocked`), `by` (top 15 of each dimension the filter leaves open: client, name, view, type, rcode, blocked; clients with `name`), `recent` (latest 100: ts, client, client_name, view, name, type, rcode, blocked), `oldest`, `retention`, `client_name`. 404 `query_log_off` without a query log |
| GET | `/stats/server` | | `{server: {rss, mem_in_use, cpu_percent, threads, pid, cache_rrsets, cache_mem, sockets_udp, sockets_tcp, recursion_highwater, tcp_highwater, version, boot_time, config_time, host: {mem_total, mem_available, load, cpus, uptime}}, named: {recursive_clients, tcp_clients, zones, cpus, worker_threads}, disks: [{path, holds, total, free}]}`; gauges over time: `/stats/timeline?key=g:named:rss` (averages, -1 without a sample) |
| GET | `/stats/breakdown` | `group` (qtype, rcode, view, zone, answer, transport, rtt, dnssec, cache, ratelimit, blocklist), `since` | `[{key, count}]` |
+4
View File
@@ -192,6 +192,10 @@ jarl option set tls 'dot { cert-file "/etc/bind/jarl/tls/ns1/fullchain.pem"; key
jarl option set listen-on 'port 853 tls dot { 10.0.10.5; }' --add
```
The Certificates page (and `jarl cert show`) says what named uses each
for: `DNS over TLS on 10.0.10.5, fd00::5 port 853`, `DNS over HTTPS on
… port 443`, or a hand-written `tls` statement no listener uses.
A certificate named uses can't be deleted or lose its `dns` use until
named no longer names its files. If you change `certificates.dir`, jarld
writes the certificates to the new place when it starts and points the
+26 -13
View File
@@ -18,6 +18,7 @@ encrypted-dns`. In the API: `/api/v1/encrypted-dns`.
```sh
jarl encrypted-dns setup --hostname dns.adbraun.lol --address 10.0.10.5 --address 2a02:8013:7f3d:10::5
jarl encrypted-dns setup --hostname dns.adbraun.lol --address 10.0.10.5=internal --address 203.0.113.5=external
```
This:
@@ -47,25 +48,37 @@ This:
`listen-on` statement at all. The first one, for DoT, would end that, so
when you have none the setup adds `listen-on { any; };` (and the same
for IPv6) to keep plain DNS where it was.
4. **adds the host name's A and AAAA records** with the chosen addresses,
in the private views that have a zone for it (`--view` to choose). In a
linked zone they are local to the view, so the name doesn't reach your
external view.
4. **adds the host names' A and AAAA records**, each address in the views
you give it: `--address 10.0.10.5=internal,lab`, or every address in
the views of `--view` (default: the private views that have a zone for
the name). A view's clients should only be given an address they can
reach, so a private address in a public view gets a warning. The
records are local to their view (in a linked zone the other views keep
theirs), and a view you no longer give an address loses it; other
records of the names stay.
Running it again replaces the earlier setup (other addresses, ports, DoT
or DoH only with `--no-doh` / `--no-dot`). `jarl encrypted-dns disable`
takes the listeners and the `tls` statement out; the certificate and the
records stay.
**Several host names** (`--hostname dns.adbraun.lol --hostname
doh.adbraun.lol`, or comma separated): each gets the records, the
certificate must be for all of them (a new one is obtained with all of
them), and the health check says when the certificate named serves misses
one. Clients may use any of them; the client hints use the first.
Running it again replaces the earlier setup: other addresses, views, host
names, **ports** (clients set up with another port need theirs changed
too), DoT or DoH only with `--no-doh` / `--no-dot`. The web UI's form
starts from the current setup, with each address's views as its records
are now. `jarl encrypted-dns disable` takes the listeners and the `tls`
statement out; the certificate and the records stay.
| Option | Default | |
|--|--|--|
| `--hostname` | | The name clients connect to; the certificate is for it |
| `--address` | | This server's addresses to listen on (repeat). Choose your own networks' addresses: a listener on a public address answers anyone who reaches it, like port 53 there |
| `--hostname` | | A name clients connect to (repeat for several); the certificate is for all of them |
| `--address` | | This server's addresses to listen on (repeat), with the views that get its records after `=` (`10.0.10.5=internal`). Choose your own networks' addresses: a listener on a public address answers anyone who reaches it, like port 53 there |
| `--certificate` | one for the host name, obtained if needed | A certificate Jarl manages |
| `--no-dot`, `--no-doh` | both on | |
| `--dot-port` | 853 | Clients expect 853 |
| `--doh-port` | 443 | Any free port; clients take it in the URL |
| `--view` | private views with the zone | Where the host name's records go |
| `--view` | private views with the zone | Where every address's records go, when `--address` gives no views |
In the web UI the same form checks the ports as you choose, offers the
free port, and obtains the certificate before setting up.
@@ -150,8 +163,8 @@ What it can say:
| Method | Path | Body |
|--------|------|------|
| GET | `/encrypted-dns` | `enabled`, `certificate`, `hostname`, `dot` and `doh` (address and port), `dot_host`, `doh_url`, `listeners`; for the form: `addresses` (this server's, with `private`), `views` (with `public`, and `has_zone` for `?hostname=`), `certificates` |
| GET | `/encrypted-dns` | `enabled`, `certificate`, `hostnames` (and `hostname`, the first), `dot` and `doh` (address and port), `address_views` (per address, the views whose records give a host name it), `dot_host`, `doh_url`, `listeners`; for the form: `addresses` (this server's, with `private`), `views` (with `public`, and `has_zone`), `certificates`. `?hostname=a,b` asks about other names |
| POST | `/encrypted-dns/check` | `{"addresses", "dot", "doh", "dot_port", "doh_port"}`: `conflicts` (address, port, what) and `free_doh_port` |
| POST | `/encrypted-dns` | `{"hostname", "certificate", "addresses", "dot", "doh", "dot_port", "doh_port", "views"}`: staged or applied like any change; 409 while the certificate hasn't been obtained |
| POST | `/encrypted-dns` | `{"hostnames", "certificate", "addresses", "address_views": {"10.0.10.5": ["internal"]}, "dot", "doh", "dot_port", "doh_port"}` (`"hostname"` and `"views"`, every address in each, work too): staged or applied like any change; 409 while the certificate hasn't been obtained |
| DELETE | `/encrypted-dns` | Turns it off |
| POST | `/encrypted-dns/health` | Checks the listeners now: `checked`, `probes` (kind, address, port, `ok`, `problem`, `warning`, certificate names, expiry and serial, `trusted`, `rcode`, `ms`). `GET /encrypted-dns` has the last check as `health` |
+32
View File
@@ -70,6 +70,7 @@ stats:
minute: 7d # per-minute counters
hour: 90d # hourly counters and top lists
day: 730d # daily counters and top lists
queries: 7d # every answer, for exploring (0: none)
top_n: 100 # names and clients kept per hour and per day
anonymize_clients: false # store 10.10.0.0/24 instead of 10.10.0.37 (IPv6: /48)
exclude_clients: [] # e.g. [10.10.0.250] to ignore a monitoring probe
@@ -192,6 +193,37 @@ jarl stats summary --since 30d -o json # export (also -o yaml)
`operator` roles include it, `dns-editor` doesn't. Statistics show client
addresses and what they look up, so grant it with care.
## Exploring
The counters and top lists say how much, not who asked for what: "100
NXDOMAIN answers in the external view" can't tell you which clients asked.
For that, jarld also keeps every answer for a while, the **query log**: time,
client, view, name, record type, result, and the blocklist (or your rules)
that blocked it. It is in the statistics database, from the response log
named writes anyway, and answers older than `retention.queries` (default
7 days) are deleted every hour; `0` keeps none.
With it, every client, name, view, record type, result and blocklist on the
Statistics page is a link to the **explore** page (`#/stats/explore`),
which shows, for what you clicked:
* the answers over time (and how many were blocked);
* the top clients, names, views, record types, results and blocklists
among them, each a link that narrows the filter further;
* the latest 100 answers, every cell a link.
So "external, NXDOMAIN" leads to the clients asking, a client to everything
it asks, one of its names to who else asks for it. Filters combine (this
client, in this view, blocked only) and each has a ✕ to take it out. A name
can be widened to the names below it (`domain=`). The Statistics page's
view switches apply here too.
`anonymize_clients` and `exclude_clients` apply to the query log as to
everything else. Its size: each answer takes about 150 to 200 bytes with
its indexes, so 50,000 answers a day for 7 days is about 60 MB, 200,000 a
day about 250 MB. The dashboard's Storage panel shows the databases' size
and how many answers the query log holds.
## Views
With views, the Statistics page has a toggle per view (and **All** and
+29
View File
@@ -277,6 +277,35 @@ type StatusResponse struct {
// EncryptedDNS is the last check of DNS over TLS and HTTPS, when it
// is set up (for those who may see options).
EncryptedDNS *EncDNSSummary `json:"encrypted_dns,omitempty"`
// Storage is the size of Jarl's databases and how full the disks Jarl
// uses are (administrators only).
Storage *Storage `json:"storage,omitempty"`
}
// Storage is the size of Jarl's databases and the disks Jarl uses.
type Storage struct {
Databases []Database `json:"databases"`
Total int64 `json:"total"` // the databases' bytes
Disks []Disk `json:"disks"`
}
// Database is one of Jarl's databases: its file with its -wal and -shm.
type Database struct {
Label string `json:"label"`
Path string `json:"path"`
Bytes int64 `json:"bytes"`
// Rows is how many answers the query log holds (statistics only).
QueryLogRows *int64 `json:"query_log_rows,omitempty"`
}
// Disk is a file system Jarl uses: what of Jarl's it holds, its size,
// what is free, and how much of it the databases take.
type Disk struct {
Path string `json:"path"`
Holds []string `json:"holds"`
Total int64 `json:"total"`
Free int64 `json:"free"`
Databases int64 `json:"databases"`
}
// CertSummary is what the dashboard reports about certificates.
+8 -1
View File
@@ -395,6 +395,9 @@ type StatsRetention struct {
Minute Duration `yaml:"minute"`
Hour Duration `yaml:"hour"`
Day Duration `yaml:"day"`
// Queries is how long the query log keeps every answer, for exploring
// (docs/stats.md#exploring); 0 keeps none.
Queries Duration `yaml:"queries"`
}
// Duration is a time.Duration written as "15s" or "12h" in YAML. Days
@@ -461,7 +464,7 @@ func Default() *Config {
LogFile: "/var/log/named/jarl-responses.log",
LogSize: "20m",
LogVersions: 2,
Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour)},
Retention: StatsRetention{Minute: Duration(7 * 24 * time.Hour), Hour: Duration(90 * 24 * time.Hour), Day: Duration(730 * 24 * time.Hour), Queries: Duration(7 * 24 * time.Hour)},
TopN: 100,
},
ACME: ACME{HTTPReq: HTTPReq{RequireHTTPS: true}, Account: ACMEAccount{Directory: "letsencrypt"},
@@ -604,6 +607,9 @@ func (s *Stats) check() error {
if r.Minute < Duration(time.Hour) || r.Hour < Duration(24*time.Hour) || r.Day < Duration(24*time.Hour) {
return fmt.Errorf("retention must keep minutes for at least 1h, hours and days for at least 1d")
}
if r.Queries < 0 || (r.Queries > 0 && r.Queries < Duration(time.Hour)) {
return fmt.Errorf("retention.queries must be at least 1h, or 0 for no query log")
}
if s.TopN < 10 || s.TopN > 10000 {
return fmt.Errorf("top_n must be between 10 and 10000")
}
@@ -754,6 +760,7 @@ stats:
minute: 7d # per-minute counters
hour: 90d # hourly counters and top lists
day: 730d # daily counters and top lists
queries: 7d # every answer, to explore (who asked what); 0: none
top_n: 100 # names and clients kept per hour and per day
anonymize_clients: false # store 10.10.0.0/24 instead of 10.10.0.37
exclude_clients: [] # e.g. [127.0.0.1] to ignore local monitoring
+183 -25
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"net/netip"
"path/filepath"
"slices"
"strconv"
"strings"
@@ -208,8 +209,10 @@ func GetStatus(st *model.State, certDir string) Status {
// Setup is what to set up.
type Setup struct {
// Hostname is the name clients connect to (the certificate's).
Hostname string `json:"hostname"`
// Hostnames are the names clients connect to; the certificate must be
// for all of them. Hostname (one) works too.
Hostnames []string `json:"hostnames,omitempty"`
Hostname string `json:"hostname,omitempty"`
// Certificate is the Jarl certificate (with the dns use) to serve.
Certificate string `json:"certificate"`
// Addresses are the addresses of this server to listen on.
@@ -218,9 +221,27 @@ type Setup struct {
DoH bool `json:"doh"`
DoHPort int `json:"doh_port"` // default 443
DoTPort int `json:"dot_port"` // default 853 (clients expect it)
// Views get the hostname's A/AAAA records (default: none). In a
// linked zone they are local to the view.
Views []string `json:"views"`
// AddressViews says, per listen address, the views where the host
// names get it (A or AAAA): a private address shouldn't reach a public
// view. Views (every address in each) works too. The records are local
// to their view in a linked zone.
AddressViews map[string][]string `json:"address_views,omitempty"`
Views []string `json:"views,omitempty"`
// PublicViews are the views the internet reaches (filled in by jarld,
// for a warning).
PublicViews map[string]bool `json:"-"`
}
// Hosts returns the setup's host names, normalized, the first first.
func (s Setup) Hosts() []string {
var out []string
for _, h := range append(append([]string{}, s.Hostnames...), s.Hostname) {
h = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(h), "."))
if h != "" && !slices.Contains(out, h) {
out = append(out, h)
}
}
return out
}
// Plan returns the requests that set up encrypted DNS as s says,
@@ -248,10 +269,15 @@ func Plan(st *model.State, s Setup, certDir string) ([]engine.Request, []string,
if s.Certificate == "" {
return nil, nil, fmt.Errorf("choose the certificate to serve")
}
host := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s.Hostname), "."))
if host == "" || !strings.Contains(host, ".") {
hosts := s.Hosts()
if len(hosts) == 0 {
return nil, nil, fmt.Errorf("give the host name clients connect to, e.g. dns.example.com")
}
for _, host := range hosts {
if !strings.Contains(host, ".") || strings.HasPrefix(host, "*") {
return nil, nil, fmt.Errorf("%q isn't a host name clients can connect to, e.g. dns.example.com", host)
}
}
var v4, v6 []string
for _, a := range s.Addresses {
ip, err := netip.ParseAddr(strings.TrimSpace(a))
@@ -325,32 +351,98 @@ func Plan(st *model.State, s Setup, certDir string) ([]engine.Request, []string,
add("listen-on", v4)
add("listen-on-v6", v6)
// The host name's addresses, in the views asked for.
fqdn := host + "."
for _, vn := range s.Views {
v := st.ViewByName(vn)
if v == nil {
// The host names' addresses, in the views asked for, per address.
want := map[string][]string{} // view -> addresses
all := append(append([]string{}, v4...), v6...)
if s.AddressViews != nil {
for a, vs := range s.AddressViews {
ip, err := netip.ParseAddr(strings.TrimSpace(a))
if err != nil || !slices.Contains(all, ip.Unmap().String()) {
return nil, nil, fmt.Errorf("address_views: %s isn't one of the addresses to listen on", a)
}
for _, vn := range vs {
want[vn] = append(want[vn], ip.Unmap().String())
}
}
} else {
for _, vn := range s.Views {
want[vn] = all
}
}
for vn, addrs := range want {
if st.ViewByName(vn) == nil {
return nil, nil, fmt.Errorf("view %q does not exist", vn)
}
z := zoneFor(st, fqdn, v.ID)
if z == nil {
notes = append(notes, fmt.Sprintf("view %s has no zone for %s, so no address records were added there; clients must find it some other way", v.Name, host))
continue
if s.PublicViews[vn] {
for _, a := range addrs {
if ip, err := netip.ParseAddr(a); err == nil && Private(ip) {
notes = append(notes, fmt.Sprintf("view %s is reachable from the internet, but gets the private address %s for %s: its clients can't reach that", vn, a, strings.Join(hosts, ", ")))
}
}
}
var local *bool
if z.LinkGroup != 0 {
local = model.Ptr(true)
}
// Ours are the addresses this setup (or the one before) gives the host
// names: a view that no longer gets one loses it; other records of the
// names stay.
ours := map[string]bool{}
for _, a := range all {
ours[a] = true
}
for _, l := range current {
if l.Managed() {
for _, a := range l.Addrs {
ours[a] = true
}
}
for _, set := range []struct {
typ string
addrs []string
}{{"A", v4}, {"AAAA", v6}} {
if len(set.addrs) == 0 {
}
names := slices.Clone(hosts)
for _, h := range st.Settings.EncryptedDNSHosts {
if !slices.Contains(names, h) {
names = append(names, h)
}
}
for _, v := range st.OrderedViews() {
for _, host := range names {
fqdn := host + "."
z := zoneFor(st, fqdn, v.ID)
wanted := want[v.Name]
if !slices.Contains(hosts, host) {
wanted = nil // a host name no longer used
}
if z == nil {
if len(wanted) > 0 {
notes = append(notes, fmt.Sprintf("view %s has no zone for %s, so no address records were added there; clients must find it some other way", v.Name, host))
}
continue
}
reqs = append(reqs, engine.NewRequest(engine.OpRRSetReplace, 0, engine.RRSetInput{ZoneID: z.ID, Name: fqdn, Type: set.typ, Data: set.addrs, Local: local}))
for _, typ := range []string{"A", "AAAA"} {
var have, next []string
for _, r := range st.RRset(z.ID, fqdn, typ) {
have = append(have, r.Data)
if !ours[r.Data] {
next = append(next, r.Data)
}
}
for _, a := range wanted {
if strings.Contains(a, ":") == (typ == "AAAA") && !slices.Contains(next, a) {
next = append(next, a)
}
}
slices.Sort(have)
slices.Sort(next)
if slices.Equal(have, next) {
continue
}
if next == nil {
next = []string{}
}
reqs = append(reqs, engine.NewRequest(engine.OpRRSetReplace, 0, engine.RRSetInput{ZoneID: z.ID, Name: fqdn, Type: typ, Data: next, Local: model.Ptr(true)}))
}
}
}
if !slices.Equal(st.Settings.EncryptedDNSHosts, hosts) {
reqs = append(reqs, engine.NewRequest(engine.OpSettingsUpdate, 0, engine.SettingsInput{EncryptedDNSHosts: &hosts}))
}
return reqs, notes, nil
}
@@ -369,6 +461,69 @@ func zoneFor(st *model.State, name string, viewID int64) *model.Zone {
return best
}
// CurrentAddressViews returns, per address, the views whose records give
// one of the host names that address (what the setup form starts from).
func CurrentAddressViews(st *model.State, hosts []string) map[string][]string {
out := map[string][]string{}
for _, v := range st.OrderedViews() {
for _, host := range hosts {
fqdn := strings.TrimSuffix(host, ".") + "."
z := zoneFor(st, fqdn, v.ID)
if z == nil {
continue
}
for _, typ := range []string{"A", "AAAA"} {
for _, r := range st.RRset(z.ID, fqdn, typ) {
if !r.Disabled && !slices.Contains(out[r.Data], v.Name) {
out[r.Data] = append(out[r.Data], v.Name)
}
}
}
}
}
return out
}
// CertUses says what of named's configuration uses the certificate whose
// files are in dir: "DNS over TLS on 10.0.10.5, fd00::5 port 853", or the
// tls statement alone when no listener uses it.
func CertUses(st *model.State, dir string) []string {
var out []string
for _, b := range TLSBlocks(st) {
if filepath.Dir(b.CertFile) != filepath.Clean(dir) {
continue
}
used := false
byKind := map[string]map[int][]string{}
for _, l := range Listeners(st) {
if l.TLS != b.Name {
continue
}
used = true
k := l.Kind()
if byKind[k] == nil {
byKind[k] = map[int][]string{}
}
byKind[k][l.EffectivePort()] = append(byKind[k][l.EffectivePort()], l.Addrs...)
}
for _, k := range []string{"dot", "doh"} {
label := map[string]string{"dot": "DNS over TLS", "doh": "DNS over HTTPS"}[k]
ports := make([]int, 0, len(byKind[k]))
for p := range byKind[k] {
ports = append(ports, p)
}
slices.Sort(ports)
for _, p := range ports {
out = append(out, fmt.Sprintf("%s on %s port %d", label, strings.Join(byKind[k][p], ", "), p))
}
}
if !used {
out = append(out, "tls statement "+b.Name+" (no listener uses it)")
}
}
return out
}
// Disable returns the requests that take the setup out: its listeners and
// tls statement. Plain listeners and the host name's records stay.
func Disable(st *model.State) []engine.Request {
@@ -383,6 +538,9 @@ func Disable(st *model.State) []engine.Request {
reqs = append(reqs, engine.NewRequest(engine.OpOptionDelete, b.OptionID, nil))
}
}
if len(reqs) > 0 && len(st.Settings.EncryptedDNSHosts) > 0 {
reqs = append(reqs, engine.NewRequest(engine.OpSettingsUpdate, 0, engine.SettingsInput{EncryptedDNSHosts: &[]string{}}))
}
return reqs
}
+67
View File
@@ -186,3 +186,70 @@ func TestPorts(t *testing.T) {
t.Errorf("free DoH port = %d, want 4443", p)
}
}
// Each address gets its own views; a view no longer given loses it, and
// records of the host names that aren't the setup's stay.
func TestSetupAddressViews(t *testing.T) {
e := newEnv(t)
e.submit(engine.NewRequest(engine.OpZoneCreate, 0, engine.ZoneInput{Name: model.Ptr("example.com"), View: model.Ptr("external"), LinkWith: "internal"}))
e.submit(engine.NewRequest(engine.OpRRSetReplace, 0, engine.RRSetInput{Zone: "example.com", View: "external", Name: "doh.example.com.", Type: "A", Data: []string{"198.51.100.9"}, Local: model.Ptr(true)}))
hosts := func(st *model.State, view string) []string {
z := st.FindZone("example.com.", st.ViewByName(view).ID)
var out []string
for _, r := range st.ZoneRecords(z.ID) {
if strings.HasPrefix(r.Name, "dns.") || strings.HasPrefix(r.Name, "doh.") {
out = append(out, strings.SplitN(r.Name, ".", 2)[0]+" "+r.Type+" "+r.Data)
}
}
return out
}
setup := Setup{Hostnames: []string{"dns.example.com", "doh.example.com"}, Certificate: "dns", DoT: true, DoH: true,
Addresses: []string{"10.0.10.5", "203.0.113.5", "10.0.10.6"},
AddressViews: map[string][]string{"10.0.10.5": {"internal"}, "203.0.113.5": {"external"}, "10.0.10.6": {"external"}},
PublicViews: map[string]bool{"external": true}}
reqs, notes, err := Plan(e.state(), setup, e.certDir)
if err != nil {
t.Fatal(err)
}
if !slices.ContainsFunc(notes, func(n string) bool {
return strings.Contains(n, "view external is reachable from the internet, but gets the private address 10.0.10.6")
}) {
t.Errorf("notes %q", notes)
}
e.submit(reqs...)
st := e.state()
if got, want := hosts(st, "internal"), []string{"dns A 10.0.10.5", "doh A 10.0.10.5"}; !slices.Equal(got, want) {
t.Errorf("internal %q, want %q", got, want)
}
// The external doh record that isn't the setup's stays.
if got, want := hosts(st, "external"), []string{"dns A 10.0.10.6", "dns A 203.0.113.5", "doh A 10.0.10.6", "doh A 198.51.100.9", "doh A 203.0.113.5"}; !slices.Equal(got, want) {
t.Errorf("external %q, want %q", got, want)
}
if !slices.Equal(st.Settings.EncryptedDNSHosts, []string{"dns.example.com", "doh.example.com"}) {
t.Errorf("hosts %q", st.Settings.EncryptedDNSHosts)
}
av := CurrentAddressViews(st, st.Settings.EncryptedDNSHosts)
if !slices.Equal(av["10.0.10.5"], []string{"internal"}) || !slices.Equal(av["10.0.10.6"], []string{"external"}) {
t.Errorf("address views %v", av)
}
uses := CertUses(st, filepath.Join(e.certDir, "dns"))
if len(uses) != 2 || !strings.HasPrefix(uses[0], "DNS over TLS on 10.0.10.5, 203.0.113.5, 10.0.10.6 port 853") || !strings.HasPrefix(uses[1], "DNS over HTTPS on") {
t.Errorf("cert uses %q", uses)
}
// 10.0.10.6 no longer in external, doh.example.com no longer a host name.
setup = Setup{Hostname: "dns.example.com", Certificate: "dns", DoT: true, Addresses: []string{"10.0.10.5", "203.0.113.5", "10.0.10.6"},
AddressViews: map[string][]string{"10.0.10.5": {"internal"}, "203.0.113.5": {"external"}}}
reqs, _, err = Plan(st, setup, e.certDir)
if err != nil {
t.Fatal(err)
}
e.submit(reqs...)
st = e.state()
if got, want := hosts(st, "external"), []string{"dns A 203.0.113.5", "doh A 198.51.100.9"}; !slices.Equal(got, want) {
t.Errorf("external after %q, want %q", got, want)
}
if got, want := hosts(st, "internal"), []string{"dns A 10.0.10.5"}; !slices.Equal(got, want) {
t.Errorf("internal after %q, want %q", got, want)
}
}
+37
View File
@@ -206,3 +206,40 @@ func describe(err error, e Endpoint) string {
}
return err.Error()
}
// RequireNames marks the probes whose certificate doesn't cover every host
// name clients may use (the first is what the probe asked for).
func (h *Health) RequireNames(hosts []string) {
for i := range h.Probes {
p := &h.Probes[i]
if p.Problem != "" || len(p.CertNames) == 0 {
continue
}
var missing []string
for _, host := range hosts {
if !nameCovered(p.CertNames, host) {
missing = append(missing, host)
}
}
if len(missing) > 0 {
p.OK = false
p.Problem = fmt.Sprintf("named serves a certificate for %s, which doesn't cover %s: clients using that name reject it", strings.Join(p.CertNames, ", "), strings.Join(missing, ", "))
}
}
}
func nameCovered(names []string, host string) bool {
host = strings.ToLower(strings.TrimSuffix(host, "."))
for _, n := range names {
n = strings.ToLower(strings.TrimSuffix(n, "."))
if n == host {
return true
}
if rest, ok := strings.CutPrefix(n, "*."); ok {
if _, parent, ok := strings.Cut(host, "."); ok && parent == rest {
return true
}
}
}
return false
}
+18
View File
@@ -113,3 +113,21 @@ func TestHealth(t *testing.T) {
t.Errorf("nothing listening: %+v", h.Probes[0])
}
}
func TestRequireNames(t *testing.T) {
h := &Health{Probes: []Probe{
{Kind: "dot", OK: true, CertNames: []string{"dns.example.com", "*.example.net"}},
{Kind: "doh", OK: false, Problem: "nothing listens", CertNames: nil},
}}
h.RequireNames([]string{"dns.example.com", "doh.example.net"})
if !h.Probes[0].OK {
t.Errorf("covered names: %+v", h.Probes[0])
}
h.RequireNames([]string{"dns.example.com", "doh.example.com"})
if p := h.Probes[0]; p.OK || !strings.Contains(p.Problem, "doesn't cover doh.example.com") {
t.Errorf("missing name: %+v", p)
}
if h.Probes[1].Problem != "nothing listens" {
t.Errorf("other problem replaced: %+v", h.Probes[1])
}
}
+2 -1
View File
@@ -25,7 +25,8 @@ func DiffOps(from, to *model.State) []model.Op {
s := to.Settings
add(OpSettingsUpdate, 0, 0, SettingsInput{DefaultTTL: &s.DefaultTTL, Nameservers: model.Ptr(append([]string{}, s.Nameservers...)), RName: &s.RName,
Refresh: &s.Refresh, Retry: &s.Retry, Expire: &s.Expire, NegativeTTL: &s.NegativeTTL, SerialPolicy: &s.SerialPolicy,
Exposure: &s.Exposure, Acknowledged: model.Ptr(slices.Clone(s.Acknowledged)), DismissedReverse: model.Ptr(slices.Clone(s.DismissedReverse))}, "restore settings")
Exposure: &s.Exposure, Acknowledged: model.Ptr(slices.Clone(s.Acknowledged)), DismissedReverse: model.Ptr(slices.Clone(s.DismissedReverse)),
EncryptedDNSHosts: model.Ptr(slices.Clone(s.EncryptedDNSHosts))}, "restore settings")
}
for _, k := range sortedKeys(to.Keys) {
+2
View File
@@ -314,6 +314,8 @@ type SettingsInput struct {
// DismissedReverse are dismissed reverse zone suggestions (without
// views).
DismissedReverse *[]string `json:"dismissed_reverse,omitempty"`
// EncryptedDNSHosts are the encrypted DNS setup's host names.
EncryptedDNSHosts *[]string `json:"encrypted_dns_hosts,omitempty"`
}
// DeleteInput carries options for delete operations.
+3
View File
@@ -1414,6 +1414,9 @@ func applySettings(st *model.State, op model.Op, c *applyCtx) error {
if in.DismissedReverse != nil {
s.DismissedReverse = slices.Clone(*in.DismissedReverse)
}
if in.EncryptedDNSHosts != nil {
s.EncryptedDNSHosts = slices.Clone(*in.EncryptedDNSHosts)
}
s.Version++
return nil
}
+4 -1
View File
@@ -275,7 +275,10 @@ type Settings struct {
// DismissedReverse are dismissed reverse zone suggestions (without
// views; see View).
DismissedReverse []string `json:"dismissed_reverse,omitempty"`
Version int `json:"version"`
// EncryptedDNSHosts are the host names clients use for DNS over TLS
// and HTTPS (docs/encrypted-dns.md); the first is checked first.
EncryptedDNSHosts []string `json:"encrypted_dns_hosts,omitempty"`
Version int `json:"version"`
}
// BlockingPaused reports whether blocking is paused, for a time or until
+18 -2
View File
@@ -9,6 +9,7 @@ import (
"jarl/internal/api"
"jarl/internal/auth"
"jarl/internal/certs"
"jarl/internal/encdns"
"jarl/internal/model"
)
@@ -21,6 +22,9 @@ type certView struct {
NextTry *time.Time `json:"next_attempt,omitempty"` // acme: when the loop obtains it next
Files *certFiles `json:"files,omitempty"` // dns use: where named reads it
Serving bool `json:"serving,omitempty"` // jarld serves it now (web use)
// NamedUse says what of named's configuration uses it: "DNS over TLS
// on 10.0.10.5 port 853" (list and get only).
NamedUse []string `json:"named_use,omitempty"`
// IssuingSince is set while Jarl is obtaining it.
IssuingSince *time.Time `json:"issuing_since,omitempty"`
}
@@ -79,9 +83,15 @@ func (s *Server) certList(w http.ResponseWriter, r *http.Request, p *auth.Princi
if err != nil {
return err
}
st, err := s.store.LoadState(r.Context())
if err != nil {
return err
}
out := []certView{}
for _, c := range all {
out = append(out, s.presentCert(c))
v := s.presentCert(c)
v.NamedUse = encdns.CertUses(st, s.cfg.Certs.FilesDir(c.Name))
out = append(out, v)
}
writeJSON(w, http.StatusOK, map[string]any{"certificates": out, "account": s.cfg.Certs.AccountInfo(r.Context())})
return nil
@@ -96,7 +106,13 @@ func (s *Server) certGet(w http.ResponseWriter, r *http.Request, p *auth.Princip
if err != nil {
return certErr(err)
}
writeJSON(w, http.StatusOK, s.presentCert(c))
st, err := s.store.LoadState(r.Context())
if err != nil {
return err
}
v := s.presentCert(c)
v.NamedUse = encdns.CertUses(st, s.cfg.Certs.FilesDir(c.Name))
writeJSON(w, http.StatusOK, v)
return nil
}
+1
View File
@@ -285,6 +285,7 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request, p *auth.Principa
resp.Backup = s.backupStatus(r.Context(), now)
resp.TOTP = s.totpOverview(r.Context())
resp.Certificates = s.certSummary(r.Context())
resp.Storage = s.storage(r.Context())
}
if p.Can(auth.Action{Resource: auth.ResOptions}) {
if h := s.encryptedDNSHealth(); h != nil {
+41 -10
View File
@@ -39,8 +39,13 @@ type encCert struct {
type encStatus struct {
encdns.Status
// Hostname is the name clients connect to (the certificate's first).
Hostname string `json:"hostname,omitempty"`
// Hostnames are the names clients connect to (the setup's, else the
// certificate's first); Hostname is the first.
Hostnames []string `json:"hostnames"`
Hostname string `json:"hostname,omitempty"`
// AddressViews are, per address, the views whose records give a host
// name that address.
AddressViews map[string][]string `json:"address_views"`
// DoTHost and DoHURL are what to give clients.
DoTHost string `json:"dot_host,omitempty"`
DoHURL string `json:"doh_url,omitempty"`
@@ -92,6 +97,15 @@ func (s *Server) encryptedDNS(w http.ResponseWriter, r *http.Request, p *auth.Pr
}
}
}
if hs := st.Settings.EncryptedDNSHosts; len(hs) > 0 {
out.Hostname = hs[0]
out.Hostnames = hs
} else if out.Hostname != "" {
out.Hostnames = []string{out.Hostname}
}
if out.Hostnames == nil {
out.Hostnames = []string{}
}
if out.Hostname != "" {
if len(out.DoT) > 0 {
out.DoTHost = out.Hostname
@@ -120,17 +134,25 @@ func (s *Server) encryptedDNS(w http.ResponseWriter, r *http.Request, p *auth.Pr
out.Addresses = append(out.Addresses, encAddress{Address: a.String(), Private: encdns.Private(a)})
}
slices.SortFunc(out.Addresses, func(a, b encAddress) int { return strings.Compare(a.Address, b.Address) })
host := strings.TrimSuffix(r.URL.Query().Get("hostname"), ".")
if host == "" {
host = out.Hostname
hosts := out.Hostnames
if q := r.URL.Query().Get("hostname"); q != "" {
hosts = nil
for _, h := range strings.Split(q, ",") {
if h = strings.TrimSuffix(strings.TrimSpace(h), "."); h != "" {
hosts = append(hosts, h)
}
}
}
// Where the records of these host names give each address now, the
// setup's or not: what the form starts from.
out.AddressViews = encdns.CurrentAddressViews(st, hosts)
for _, x := range s.eng.Exposures(st) {
if x.View == "" {
continue
}
v := encView{Name: x.View, Public: x.Public}
if host != "" {
if vv := st.ViewByName(x.View); vv != nil {
if vv := st.ViewByName(x.View); vv != nil {
for _, host := range hosts {
for _, z := range st.ZonesInView(vv.ID) {
if z.Type == model.ZonePrimary && (host+"." == z.Name || strings.HasSuffix(host+".", "."+z.Name)) {
v.HasZone = true
@@ -204,13 +226,17 @@ func (s *Server) encryptedDNSSetup(w http.ResponseWriter, r *http.Request, p *au
return badRequest("certificate %s doesn't have the dns use, so named can't read it: jarl cert update %s --use dns (and keep its other uses)", c.Name, c.Name)
case c.NotAfter == nil:
return conflict("certificate %s hasn't been obtained yet (%s); set up encrypted DNS once it has", c.Name, s.cfg.Certs.StatusOf(c).Message)
case in.Hostname != "" && !certs.Covers(c.Names, []string{in.Hostname}):
return badRequest("certificate %s is for %s, not %s: clients would reject it", c.Name, strings.Join(c.Names, ", "), in.Hostname)
case len(in.Hosts()) > 0 && !certs.Covers(c.Names, in.Hosts()):
return badRequest("certificate %s is for %s, not all of %s: clients would reject it (add the names to it: jarl cert update %s --domain …)", c.Name, strings.Join(c.Names, ", "), strings.Join(in.Hosts(), ", "), c.Name)
}
st, err := s.store.LoadState(r.Context())
if err != nil {
return err
}
in.PublicViews = map[string]bool{}
for _, x := range s.eng.Exposures(st) {
in.PublicViews[x.View] = x.Public
}
reqs, notes, err := encdns.Plan(st, in, s.cfg.Certs.FilesDir(""))
if err != nil {
return badRequest("%v", err)
@@ -222,7 +248,7 @@ func (s *Server) encryptedDNSSetup(w http.ResponseWriter, r *http.Request, p *au
if in.DoH {
what = append(what, "DNS over HTTPS")
}
return s.mutateNotes(w, r, p, "set up "+strings.Join(what, " and ")+" for "+in.Hostname, reqs, nil, notes)
return s.mutateNotes(w, r, p, "set up "+strings.Join(what, " and ")+" for "+strings.Join(in.Hosts(), ", "), reqs, nil, notes)
}
// encryptedDNSDisable: DELETE /encrypted-dns.
@@ -274,7 +300,12 @@ func (s *Server) checkEncryptedDNS(ctx context.Context) *encdns.Health {
var h *encdns.Health
if st.Enabled {
if host, serial := s.encTarget(ctx, st); host != "" {
hosts := state.Settings.EncryptedDNSHosts
if len(hosts) > 0 {
host = hosts[0]
}
h = encdns.Check(ctx, st, host, serial, s.encRoots)
h.RequireNames(append([]string{host}, hosts...))
}
}
s.encHealth.Lock()
+56
View File
@@ -286,3 +286,59 @@ func (s *Server) statsServer(w http.ResponseWriter, r *http.Request, p *auth.Pri
writeJSON(w, http.StatusOK, out)
return nil
}
// statsExplore: GET /stats/explore?client=&name=&domain=&view=&type=&rcode=
// &blocked=&since=24h explores the query log (docs/stats.md#exploring):
// totals, a series, the top values of each dimension left open, and the
// latest answers, with client host names.
func (s *Server) statsExplore(w http.ResponseWriter, r *http.Request, p *auth.Principal) error {
q, err := s.statsQuerier(r, p)
if err != nil {
return err
}
since, err := statsSince(r)
if err != nil {
return err
}
v := r.URL.Query()
f := stats.ExploreFilter{Client: v.Get("client"), Name: v.Get("name"), Domain: v.Get("domain"), View: v.Get("view"),
Type: v.Get("type"), RCode: v.Get("rcode"), Blocked: v.Get("blocked"), Since: since}
ex, err := q.Explore(r.Context(), f, 15, 100)
if errors.Is(err, stats.ErrNoQueryLog) {
return &httpError{http.StatusNotFound, "query_log_off", err.Error()}
}
if err != nil {
return err
}
if err := s.nameClients(r, p, ex.By["client"]); err != nil {
return err
}
// The recent answers' clients, named once each.
var es []stats.Entry
seen := map[string]int{}
for _, row := range ex.Recent {
if _, ok := seen[row.Client]; !ok {
seen[row.Client] = len(es)
es = append(es, stats.Entry{Key: row.Client})
}
}
if err := s.nameClients(r, p, es); err != nil {
return err
}
for i := range ex.Recent {
ex.Recent[i].ClientName = es[seen[ex.Recent[i].Client]].Name
}
// The filter's client, named too.
out := struct {
*stats.Exploration
ClientName string `json:"client_name,omitempty"`
}{Exploration: ex}
if f.Client != "" {
one := []stats.Entry{{Key: f.Client}}
if err := s.nameClients(r, p, one); err == nil {
out.ClientName = one[0].Name
}
}
writeJSON(w, http.StatusOK, out)
return nil
}
+72
View File
@@ -0,0 +1,72 @@
package server
import (
"context"
"os"
"path/filepath"
"slices"
"jarl/internal/api"
"jarl/internal/stats"
)
// storage reports the size of Jarl's databases and how full the disks
// Jarl uses are (the dashboard's Storage card).
func (s *Server) storage(ctx context.Context) *api.Storage {
out := &api.Storage{Databases: []api.Database{}, Disks: []api.Disk{}}
dbFile := func(label, path string) *api.Database {
if path == "" || path == ":memory:" {
return nil
}
d := api.Database{Label: label, Path: path}
for _, suffix := range []string{"", "-wal", "-shm"} {
if fi, err := os.Stat(path + suffix); err == nil {
d.Bytes += fi.Size()
}
}
out.Databases = append(out.Databases, d)
out.Total += d.Bytes
return &out.Databases[len(out.Databases)-1]
}
labels := []string{}
dirs := map[string]string{}
add := func(label, dir string) {
if dir != "" {
labels = append(labels, label)
dirs[label] = dir
}
}
const jarlDB, statsDB = "Jarl's database", "statistics database"
if p := s.store.Path(); p != "" {
dbFile(jarlDB, p)
add(jarlDB, filepath.Dir(p))
}
if s.cfg.Stats != nil {
if p := s.cfg.Stats.Store().Path(); p != "" && p != ":memory:" {
if d := dbFile(statsDB, p); d != nil && s.cfg.Stats.Options().RetainQueries > 0 {
if n, err := s.cfg.Stats.Store().QueryLogSize(ctx); err == nil {
d.QueryLogRows = &n
}
}
add(statsDB, filepath.Dir(p))
}
if lf := s.cfg.Stats.Options().LogFile; lf != "" {
add("named's logs", filepath.Dir(lf))
}
}
paths := s.eng.Config().Paths
add("zone files", paths.ZoneDir)
add("named.conf", paths.ConfigDir)
add("Jarl's cache and blocklists", paths.CacheDir)
add("backups", s.cfg.BackupDir)
for _, d := range stats.DiskUsage(labels, dirs) {
disk := api.Disk{Path: d.Path, Holds: d.Holds, Total: d.Total, Free: d.Free}
for _, db := range out.Databases {
if slices.Contains(d.Holds, db.Label) {
disk.Databases += db.Bytes
}
}
out.Disks = append(out.Disks, disk)
}
return out
}
+1
View File
@@ -195,6 +195,7 @@ func (s *Server) routes() {
m.HandleFunc("GET "+p+"/stats/heatmap", s.authed(s.statsHeatmap))
m.HandleFunc("GET "+p+"/stats/breakdown", s.authed(s.statsBreakdown))
m.HandleFunc("GET "+p+"/stats/server", s.authed(s.statsServer))
m.HandleFunc("GET "+p+"/stats/explore", s.authed(s.statsExplore))
m.HandleFunc(p+"/", func(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "not_found", "no such API endpoint: "+r.Method+" "+r.URL.Path)
})
+55
View File
@@ -121,3 +121,58 @@ func TestStatsAPI(t *testing.T) {
vtok := e.do("POST", "/users/"+itoa(viewerID)+"/tokens", e.token, map[string]any{"name": "t"}, 201)["token"].(string)
e.do("GET", "/stats/summary", vtok, nil, 200)
}
func TestStatsExploreAPI(t *testing.T) {
ctx := context.Background()
ss, err := stats.OpenStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer ss.Close()
now := time.Now().Unix()
b := &stats.Batch{}
b.AddQueries([]stats.QueryRow{
{Time: now - 60, Client: "10.0.0.7", View: "internal", Name: "www.example.com", Type: "A", RCode: "NOERROR"},
{Time: now - 50, Client: "10.0.0.7", View: "internal", Name: "ads.tracker.test", Type: "A", RCode: "NXDOMAIN", Blocked: "hagezi"},
{Time: now - 40, Client: "198.51.100.9", View: "external", Name: "nope.example.com", Type: "A", RCode: "NXDOMAIN"},
})
if err := ss.Commit(ctx, b); err != nil {
t.Fatal(err)
}
opts := stats.Options{QueryLog: true, StatisticsChannel: "127.0.0.1:1", TopN: 10, RetainMinute: time.Hour, RetainHour: 24 * time.Hour, RetainDay: 24 * time.Hour, RetainQueries: 7 * 24 * time.Hour}
c := stats.NewCollector(ss, opts, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
e := newAPIEnv(t, c)
e.do("POST", "/zones", e.token, map[string]any{"name": "example.com", "records": []map[string]any{{"name": "ns1", "type": "A", "data": "192.0.2.1"}, {"name": "laptop", "type": "A", "data": "10.0.0.7"}}}, 201)
ex := e.do("GET", "/stats/explore?rcode=NXDOMAIN&since=1h", e.token, nil, 200)
if ex["total"].(float64) != 2 || ex["blocked"].(float64) != 1 {
t.Errorf("NXDOMAIN: %v", ex)
}
clients := ex["by"].(map[string]any)["client"].([]any)
if len(clients) != 2 {
t.Fatalf("clients %v", clients)
}
named := false
for _, c := range clients {
m := c.(map[string]any)
named = named || m["key"] == "10.0.0.7" && m["name"] == "laptop.example.com"
}
if !named {
t.Errorf("client names %v", clients)
}
ex = e.do("GET", "/stats/explore?client=10.0.0.7", e.token, nil, 200)
recent := ex["recent"].([]any)
if ex["total"].(float64) != 2 || ex["client_name"] != "laptop.example.com" || len(recent) != 2 || recent[0].(map[string]any)["client_name"] != "laptop.example.com" {
t.Errorf("laptop: %v", ex)
}
if ex := e.do("GET", "/stats/explore?client=10.0.0.7&views=external", e.token, nil, 200); ex["total"].(float64) != 0 {
t.Errorf("laptop in external only: %v", ex["total"])
}
// The dashboard reports the databases' size.
st := e.do("GET", "/status", e.token, nil, 200)
sg, ok := st["storage"].(map[string]any)
if !ok || len(sg["databases"].([]any)) == 0 {
t.Errorf("storage %v", st["storage"])
}
}
+42 -12
View File
@@ -56,9 +56,12 @@ type Options struct {
RetainMinute time.Duration
RetainHour time.Duration
RetainDay time.Duration
Anonymize bool
Exclude []netip.Prefix
Location *time.Location // for days and log timestamps (default: local time)
// RetainQueries is how long the query log keeps every answer (0: no
// query log; it needs QueryLog).
RetainQueries time.Duration
Anonymize bool
Exclude []netip.Prefix
Location *time.Location // for days and log timestamps (default: local time)
// MainConfig is named's main configuration file; it tells named's
// process apart from other named processes (for its memory and CPU).
MainConfig string
@@ -87,9 +90,10 @@ type Status struct {
Warnings []string `json:"warnings"`
FirstData *time.Time `json:"first_data,omitempty"`
Retention struct {
Minute string `json:"minute"`
Hour string `json:"hour"`
Day string `json:"day"`
Minute string `json:"minute"`
Hour string `json:"hour"`
Day string `json:"day"`
Queries string `json:"queries"` // the query log ("" when off)
} `json:"retention"`
}
@@ -119,9 +123,11 @@ type Collector struct {
recentL []int64 // response log lines per minute, last 10 minutes
linesMin int64
ensureAt time.Time
// rewritten holds queries named blocked (client#port|name) until their
// response line arrives.
rewritten map[string]bool
// rewritten holds queries named blocked (client#port|name), and what
// blocked them, until their response line arrives.
rewritten map[string]string
// pending are answers for the query log, written at the next flush.
pending []QueryRow
// viewsSince is set once per-view counters exist (meta views_since).
viewsSince bool
viewsSinceAt int64
@@ -137,6 +143,10 @@ type Collector struct {
// maxKeys bounds the per-hour maps; rarely seen keys are dropped first.
const maxKeys = 200_000
// maxPending bounds the answers waiting for the query log between flushes
// (a minute); past it they are counted but not logged.
const maxPending = 500_000
// NewCollector creates a collector. Run starts it.
func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger) *Collector {
if opts.Location == nil {
@@ -154,6 +164,9 @@ func NewCollector(store *Store, opts Options, zones ZoneIndex, log *slog.Logger)
c.status.Retention.Minute = humanDuration(opts.RetainMinute)
c.status.Retention.Hour = humanDuration(opts.RetainHour)
c.status.Retention.Day = humanDuration(opts.RetainDay)
if opts.QueryLog && opts.RetainQueries > 0 {
c.status.Retention.Queries = humanDuration(opts.RetainQueries)
}
return c
}
@@ -362,7 +375,8 @@ func (c *Collector) handleLine(line string) {
}
return
}
blocked := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
blockedBy := c.rewritten[ev.Client.String()+"#"+ev.Port+"|"+ev.Name]
blocked := blockedBy != ""
if blocked {
delete(c.rewritten, ev.Client.String()+"#"+ev.Port+"|"+ev.Name)
}
@@ -405,6 +419,9 @@ func (c *Collector) handleLine(line string) {
}
add(KindName, ev.Name)
add(KindClient, client)
if c.opts.RetainQueries > 0 && len(c.pending) < maxPending {
c.pending = append(c.pending, QueryRow{Time: ev.Time.Unix(), Client: client, View: view, Name: ev.Name, Type: ev.Type, RCode: ev.RCode, Blocked: blockedBy})
}
switch {
case blocked:
// Counted by handleRewrite; not a name that doesn't exist.
@@ -445,9 +462,9 @@ func (c *Collector) handleRewrite(rw Rewrite) {
if c.rewritten == nil || len(c.rewritten) > 10_000 {
// Rewrites whose response line never came (named only logs
// responses while response logging is on) don't pile up.
c.rewritten = map[string]bool{}
c.rewritten = map[string]string{}
}
c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = true
c.rewritten[rw.Client.String()+"#"+rw.Port+"|"+rw.Name] = policySource(rw.Via)
minute, hour := c.minuteOf(rw.Time), c.hourOf(rw.Time)
view := rw.View
if view == "" {
@@ -639,6 +656,8 @@ func (c *Collector) flush(ctx context.Context, final bool) {
pos, _ := json.Marshal(c.tail.Position())
b.SetMeta("tail", string(pos))
}
b.AddQueries(c.pending)
c.pending = nil
c.updateWarnings()
c.mu.Unlock()
@@ -737,6 +756,17 @@ func (c *Collector) prune(ctx context.Context) {
c.log.Debug("statistics: pruned", "resolution", res, "rows", n)
}
}
// The query log keeps answers for RetainQueries; with no query log,
// none at all (it may have been turned off).
keep := c.opts.RetainQueries
if !c.opts.QueryLog {
keep = 0
}
if n, err := c.store.PruneQueries(ctx, now-int64(keep.Seconds())); err != nil {
c.log.Warn("statistics: pruning the query log failed", "err", err)
} else if n > 0 {
c.log.Debug("statistics: pruned the query log", "rows", n)
}
}
func humanDuration(d time.Duration) string {
+199
View File
@@ -0,0 +1,199 @@
package stats
import (
"context"
"fmt"
"strings"
"time"
)
// Exploring the query log (docs/stats.md#exploring): every answer of the
// last days (stats.query_retention), filtered by any mix of client, name,
// view, type, result and blocking, broken down by each of them.
// ExploreFilter narrows the query log. Empty fields don't filter.
type ExploreFilter struct {
Client string // a client key, as the top lists show it
Name string // exactly this name
Domain string // this name or any below it
View string
Type string
RCode string
// Blocked is "yes" (blocked by anything), "no", or a blocklist's name
// ("rules" for your own rules).
Blocked string
Since time.Duration // how far back (default 24h)
}
// Dimensions are what the query log breaks down by, in the order shown.
var Dimensions = []string{"client", "name", "view", "type", "rcode", "blocked"}
// Exploration is what the query log says for a filter.
type Exploration struct {
From time.Time `json:"from"`
To time.Time `json:"to"`
Total int64 `json:"total"`
// Blocked counts the blocked answers among them.
Blocked int64 `json:"blocked"`
// Series are the answers (and blocked ones) per Step seconds.
Step int64 `json:"step"`
Series []ExplorePoint `json:"series"`
// By are the top values of each dimension the filter leaves open.
By map[string][]Entry `json:"by"`
// Recent are the latest answers.
Recent []QueryRow `json:"recent"`
// Oldest is the first answer the query log still has.
Oldest *time.Time `json:"oldest,omitempty"`
// Retention is how long it keeps them.
Retention string `json:"retention"`
}
// ExplorePoint is one step of the series.
type ExplorePoint struct {
Time int64 `json:"t"`
Count int64 `json:"count"`
Blocked int64 `json:"blocked"`
}
// ErrNoQueryLog is returned when the query log is off.
var ErrNoQueryLog = fmt.Errorf("the query log is off: it needs stats.query_log and a stats.query_retention above 0 in /etc/jarl/jarl.yaml (docs/stats.md#exploring)")
// where builds the SQL condition and arguments for a filter.
func (q *Querier) where(f ExploreFilter, from int64) (string, []any) {
conds := []string{"ts >= ?"}
args := []any{from}
eq := func(col, v string) {
if v != "" {
conds = append(conds, col+" = ?")
args = append(args, v)
}
}
eq("client", f.Client)
eq("name", strings.ToLower(strings.TrimSuffix(f.Name, ".")))
eq("view", f.View)
eq("type", strings.ToUpper(f.Type))
eq("rcode", strings.ToUpper(f.RCode))
if d := strings.ToLower(strings.TrimSuffix(f.Domain, ".")); d != "" {
conds = append(conds, "(name = ? OR (length(name) > length(?) AND substr(name, length(name) - length(?)) = ?))")
args = append(args, d, d, d, "."+d)
}
switch f.Blocked {
case "":
case "yes":
conds = append(conds, "blocked != ''")
case "no":
conds = append(conds, "blocked = ''")
default:
eq("blocked", f.Blocked)
}
if len(q.Views) > 0 {
marks := strings.TrimSuffix(strings.Repeat("?,", len(q.Views)), ",")
conds = append(conds, "view IN ("+marks+")")
for _, v := range q.Views {
args = append(args, v)
}
}
return strings.Join(conds, " AND "), args
}
// Explore answers a filter: totals, a series, the top values of each open
// dimension (topN each) and the latest answers (recent of them).
func (q *Querier) Explore(ctx context.Context, f ExploreFilter, topN, recent int) (*Exploration, error) {
if !q.Opts.QueryLog || q.Opts.RetainQueries <= 0 {
return nil, ErrNoQueryLog
}
now := q.Now()
since := f.Since
if since <= 0 {
since = 24 * time.Hour
}
if since > q.Opts.RetainQueries {
since = q.Opts.RetainQueries
}
from := now.Add(-since).Unix()
out := &Exploration{From: time.Unix(from, 0), To: now, By: map[string][]Entry{}, Series: []ExplorePoint{}, Recent: []QueryRow{},
Retention: humanDuration(q.Opts.RetainQueries)}
cond, args := q.where(f, from)
db := q.Store.db
if err := db.QueryRowContext(ctx, `SELECT count(*), coalesce(sum(blocked != ''), 0) FROM queries WHERE `+cond, args...).Scan(&out.Total, &out.Blocked); err != nil {
return nil, err
}
// The series: about 100 steps over the range, in whole minutes.
step := int64(since.Seconds()) / 100
step = max(60, step-step%60)
out.Step = step
rows, err := db.QueryContext(ctx, `SELECT ts - ts % ? AS t, count(*), sum(blocked != '') FROM queries WHERE `+cond+` GROUP BY t ORDER BY t`, append([]any{step}, args...)...)
if err != nil {
return nil, err
}
got := map[int64]ExplorePoint{}
for rows.Next() {
var p ExplorePoint
if err := rows.Scan(&p.Time, &p.Count, &p.Blocked); err != nil {
rows.Close()
return nil, err
}
got[p.Time] = p
}
rows.Close()
for t := from - from%step; t <= now.Unix(); t += step {
p := got[t]
p.Time = t
out.Series = append(out.Series, p)
}
open := map[string]bool{"client": f.Client == "", "name": f.Name == "", "view": f.View == "", "type": f.Type == "", "rcode": f.RCode == "", "blocked": f.Blocked == "" || f.Blocked == "yes"}
for _, dim := range Dimensions {
if !open[dim] {
continue
}
c := cond
if dim == "blocked" {
c += " AND blocked != ''"
}
rows, err := db.QueryContext(ctx, `SELECT `+dim+`, count(*) AS n FROM queries WHERE `+c+` GROUP BY `+dim+` ORDER BY n DESC, `+dim+` LIMIT ?`, append(args, topN)...)
if err != nil {
return nil, err
}
es := []Entry{}
for rows.Next() {
var e Entry
if err := rows.Scan(&e.Key, &e.Count); err != nil {
rows.Close()
return nil, err
}
es = append(es, e)
}
rows.Close()
out.By[dim] = es
}
rows, err = db.QueryContext(ctx, `SELECT ts, client, view, name, type, rcode, blocked FROM queries WHERE `+cond+` ORDER BY ts DESC, rowid DESC LIMIT ?`, append(args, recent)...)
if err != nil {
return nil, err
}
for rows.Next() {
var r QueryRow
if err := rows.Scan(&r.Time, &r.Client, &r.View, &r.Name, &r.Type, &r.RCode, &r.Blocked); err != nil {
rows.Close()
return nil, err
}
out.Recent = append(out.Recent, r)
}
rows.Close()
var oldest *int64
if err := db.QueryRowContext(ctx, `SELECT min(ts) FROM queries`).Scan(&oldest); err == nil && oldest != nil {
t := time.Unix(*oldest, 0)
out.Oldest = &t
}
return out, nil
}
// QueryLogSize returns how many answers the query log has.
func (s *Store) QueryLogSize(ctx context.Context) (int64, error) {
var n int64
err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM queries`).Scan(&n)
return n, err
}
+98
View File
@@ -0,0 +1,98 @@
package stats
import (
"context"
"fmt"
"io"
"log/slog"
"path/filepath"
"testing"
"time"
)
func TestExplore(t *testing.T) {
ctx := context.Background()
loc := time.UTC
store, err := OpenStore(filepath.Join(t.TempDir(), "stats.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
clk := &clock{time.Date(2026, 10, 13, 12, 0, 30, 0, loc)}
opts := Options{QueryLog: true, LogFile: "unused", StatisticsChannel: "unused", TopN: 10, RetainMinute: time.Hour, RetainHour: time.Hour, RetainDay: time.Hour,
RetainQueries: 7 * 24 * time.Hour, Location: loc}
c := NewCollector(store, opts, func() map[string][]string { return nil }, slog.New(slog.NewTextHandler(io.Discard, nil)))
c.now = clk.now
at := clk.t.Add(-10 * time.Minute)
// The phone: lookups, an NXDOMAIN, a blocked tracker.
c.HandleLine(logLine(at, "10.0.0.5", "internal", "www.adbraun.lol", "A", "NOERROR"))
c.HandleLine(logLine(at, "10.0.0.5", "internal", "nas.adbraun.lol", "AAAA", "NOERROR"))
c.HandleLine(logLine(at, "10.0.0.5", "internal", "typo.adbraun.lol", "A", "NXDOMAIN"))
c.HandleLine(fmt.Sprintf("%s client @0x7f0000000000 10.0.0.5#5353 (t.tracker.example): view internal: rpz QNAME NXDOMAIN rewrite t.tracker.example/A/IN via t.tracker.example.hagezi.blocklist.jarl.invalid\n", at.Format(logTimeLayout)))
c.HandleLine(logLine(at, "10.0.0.5", "internal", "t.tracker.example", "A", "NXDOMAIN"))
// A scanner in the external view, NXDOMAIN twice.
for i := 0; i < 2; i++ {
c.HandleLine(logLine(at, "198.51.100.7", "external", "nope.adbraun.lol", "A", "NXDOMAIN"))
}
c.flush(ctx, true)
q := &Querier{Store: store, Opts: opts, Now: clk.now}
ex, err := q.Explore(ctx, ExploreFilter{View: "external", RCode: "NXDOMAIN"}, 10, 50)
if err != nil {
t.Fatal(err)
}
if ex.Total != 2 || len(ex.By["client"]) != 1 || ex.By["client"][0].Key != "198.51.100.7" || ex.By["view"] != nil {
t.Errorf("external NXDOMAIN: %+v", ex)
}
// What else does the phone do?
ex, err = q.Explore(ctx, ExploreFilter{Client: "10.0.0.5"}, 10, 50)
if err != nil {
t.Fatal(err)
}
if ex.Total != 4 || ex.Blocked != 1 || len(ex.Recent) != 4 {
t.Errorf("phone: total %d blocked %d recent %d", ex.Total, ex.Blocked, len(ex.Recent))
}
if b := ex.By["blocked"]; len(b) != 1 || b[0].Key != "hagezi" || b[0].Count != 1 {
t.Errorf("blocked by %+v", b)
}
if r := ex.By["rcode"]; len(r) != 2 || r[0].Key != "NOERROR" || r[0].Count != 2 {
t.Errorf("rcode %+v", r)
}
var sum int64
for _, p := range ex.Series {
sum += p.Count
}
if sum != 4 || ex.Step != 60*int64(24*60/100) {
t.Errorf("series sum %d step %d", sum, ex.Step)
}
// A domain and what's below it.
ex, _ = q.Explore(ctx, ExploreFilter{Domain: "adbraun.lol"}, 10, 50)
if ex.Total != 5 {
t.Errorf("domain adbraun.lol: %d", ex.Total)
}
ex, _ = q.Explore(ctx, ExploreFilter{Name: "nas.adbraun.lol", Type: "aaaa"}, 10, 50)
if ex.Total != 1 || ex.Recent[0].Client != "10.0.0.5" {
t.Errorf("nas AAAA: %+v", ex)
}
ex, _ = q.Explore(ctx, ExploreFilter{Blocked: "no", Client: "10.0.0.5"}, 10, 50)
if ex.Total != 3 {
t.Errorf("not blocked: %d", ex.Total)
}
// Limited to a view (the Statistics page's view switches).
ex, _ = q.ForViews([]string{"internal"}).Explore(ctx, ExploreFilter{RCode: "NXDOMAIN"}, 10, 50)
if ex.Total != 2 {
t.Errorf("internal NXDOMAIN: %d", ex.Total)
}
// Pruning: past the retention they go.
clk.t = clk.t.Add(8 * 24 * time.Hour)
c.prune(ctx)
if n, _ := store.QueryLogSize(ctx); n != 0 {
t.Errorf("after pruning: %d rows", n)
}
// Off: an error that says how to turn it on.
q.Opts.RetainQueries = 0
if _, err := q.Explore(ctx, ExploreFilter{}, 10, 50); err != ErrNoQueryLog {
t.Errorf("off: %v", err)
}
}
+62
View File
@@ -45,6 +45,21 @@ var schema = []string{
PRIMARY KEY (res, ts, kind, key)
) WITHOUT ROWID;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);`,
// 2: the query log, every answer for a while (stats.query_retention),
// for drilling down: what a client asked, who asked for a name
// (docs/stats.md#exploring).
`CREATE TABLE queries (
ts INTEGER NOT NULL,
client TEXT NOT NULL,
view TEXT NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL,
rcode TEXT NOT NULL,
blocked TEXT NOT NULL DEFAULT ''
);
CREATE INDEX queries_ts ON queries (ts);
CREATE INDEX queries_client ON queries (client, ts);
CREATE INDEX queries_name ON queries (name, ts);`,
}
// OpenStore opens (creating if needed) the statistics database. Use
@@ -123,8 +138,25 @@ type Batch struct {
counters []counterRow
tops []topsWrite
meta map[string]string
queries []QueryRow
}
// QueryRow is one answer in the query log.
type QueryRow struct {
Time int64 `json:"ts"` // Unix seconds
Client string `json:"client"`
View string `json:"view"` // "_default" without views
Name string `json:"name"`
Type string `json:"type"`
RCode string `json:"rcode"`
Blocked string `json:"blocked,omitempty"` // the blocklist (or "rules") that blocked it
// ClientName is the client's host name (set by the API).
ClientName string `json:"client_name,omitempty"`
}
// AddQueries adds answers to the query log.
func (b *Batch) AddQueries(rows []QueryRow) { b.queries = append(b.queries, rows...) }
type counterRow struct {
res string
ts int64
@@ -220,9 +252,39 @@ func (s *Store) Commit(ctx context.Context, b *Batch) error {
return err
}
}
if len(b.queries) > 0 {
st, err := tx.PrepareContext(ctx, `INSERT INTO queries (ts, client, view, name, type, rcode, blocked) VALUES (?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
for _, q := range b.queries {
if _, err := st.ExecContext(ctx, q.Time, q.Client, q.View, q.Name, q.Type, q.RCode, q.Blocked); err != nil {
st.Close()
return err
}
}
st.Close()
}
return tx.Commit()
}
// PruneQueries deletes the query log's answers older than before (Unix
// time), a slice at a time so the database isn't locked for long.
func (s *Store) PruneQueries(ctx context.Context, before int64) (int64, error) {
var n int64
for {
r, err := s.db.ExecContext(ctx, `DELETE FROM queries WHERE rowid IN (SELECT rowid FROM queries WHERE ts < ? LIMIT 50000)`, before)
if err != nil {
return n, err
}
c, _ := r.RowsAffected()
n += c
if c < 50000 {
return n, nil
}
}
}
// Meta returns a stored value ("" if unset).
func (s *Store) Meta(ctx context.Context, key string) (string, error) {
var v string
+9
View File
@@ -29,6 +29,15 @@ type Store struct {
// DB exposes the underlying database (for backups and tests).
func (s *Store) DB() *sql.DB { return s.db }
// Path returns the database file ("" in memory).
func (s *Store) Path() string {
var file string
if err := s.db.QueryRow(`SELECT file FROM pragma_database_list WHERE name = 'main'`).Scan(&file); err != nil {
return ""
}
return file
}
// Open opens (creating if needed) the database at path and applies
// migrations. Use ":memory:" for a throwaway in-memory database.
func Open(path string) (*Store, error) {
+2
View File
@@ -17,6 +17,7 @@ import * as users from "./pages/users.js";
import * as account from "./pages/account.js";
import * as search from "./pages/search.js";
import * as stats from "./pages/stats.js";
import * as explore from "./pages/explore.js";
import * as blocking from "./pages/blocking.js";
import * as exposure from "./pages/exposure.js";
import * as certificates from "./pages/certificates.js";
@@ -39,6 +40,7 @@ const routes = [
{ pattern: /^account$/, page: account, nav: "account" },
{ pattern: /^search$/, page: search, nav: "" },
{ pattern: /^stats$/, page: stats, nav: "stats" },
{ pattern: /^stats\/explore$/, page: explore, nav: "stats" },
{ pattern: /^blocking$/, page: blocking, nav: "blocking" },
{ pattern: /^exposure$/, page: exposure, nav: "exposure" },
{ pattern: /^certificates$/, page: certificates, nav: "certificates" },
@@ -33,7 +33,8 @@ export async function render(el) {
{ label: "Names", render: (c) => h("div", { class: "mono small" }, (c.names && c.names.length ? c.names : c.domains).map((n) => h("div", null, n))) },
{ label: "Used for", render: (c) => [
c.uses.includes("web") ? badge(c.serving ? "web UI (serving)" : "web UI", c.serving ? "ok" : "") : null, " ",
c.uses.includes("dns") ? badge("named", "") : null,
c.uses.includes("dns") ? [badge("named", ""), (c.named_use || []).map((u) => h("div", { class: "small muted", title: u }, u.replace("DNS over TLS on ", "DoT ").replace("DNS over HTTPS on ", "DoH "))),
!(c.named_use || []).length ? h("div", { class: "small muted" }, "not used by named yet") : null] : null,
!c.uses.length ? h("span", { class: "muted" }, "nothing yet") : null] },
{ label: "Source", render: (c) => c.source === "acme" ? h("span", null, "Jarl", h("div", { class: "small muted" }, "challenge " + c.challenge)) : h("span", null, "your files", h("div", { class: "small muted mono" }, c.cert_file)) },
{ label: "Expires", render: (c) => c.not_after ? h("span", null, time(c.not_after), c.source === "acme" && c.next_attempt ? h("div", { class: "small muted" }, "renews " + fmtTime(c.next_attempt)) : null) : h("span", { class: "muted" }, "–") },
@@ -79,6 +80,7 @@ function details(c) {
modal("Certificate " + c.name, h("div", null,
h("dl", { class: "kv" }, rows.map(([k, v]) => [h("dt", null, k), h("dd", null, v)])),
c.files ? h("div", null, h("h3", null, "For named"),
(c.named_use || []).length ? h("ul", { class: "small" }, c.named_use.map((u) => h("li", null, u))) : h("p", { class: "small muted" }, "named doesn't use it yet (Encrypted DNS sets that up)."),
h("p", { class: "small muted" }, "Point a tls block in named's configuration at these files; Jarl rewrites them on renewal and has named reread them."),
copyBox(c.files.chain), copyBox(c.files.key)) : null), [{ label: "Close", value: null }], { wide: true });
}
@@ -18,6 +18,7 @@ export async function render(el) {
st.backup ? backupCard(st.backup) : null,
exposure ? exposureCard(exposure) : null,
st.certificates ? certCard(st.certificates) : null,
st.storage ? storageCard(st.storage) : null,
st.encrypted_dns ? h("a", { class: "card link", href: "#/encrypted-dns" }, h("div", { class: "label" }, "Encrypted DNS"),
h("div", { class: "value" }, st.encrypted_dns.ok ? badge("working", "ok") : badge("not working", "err")),
h("div", { class: "sub" }, "checked ", time(st.encrypted_dns.checked))) : null,
@@ -100,6 +101,7 @@ export async function render(el) {
h("h2", null, "Configuration checks"),
real.length ? problemsList(real) : h("div", { class: "msg ok" }, "No problems found in the current configuration."),
problems.length > real.length ? h("details", null, h("summary", { class: "muted" }, plural(problems.length - real.length, "note")), problemsList(problems.filter((p) => p.severity === "info"))) : null));
if (st.storage) parts.push(storagePanel(st.storage));
if (s.paths) {
parts.push(h("div", { class: "panel" }, h("h2", null, "Files"), h("dl", { class: "kv" },
h("dt", null, "named's main config"), h("dd", { class: "mono" }, s.main_config || "–"),
@@ -140,3 +142,44 @@ function certCard(c) {
function card(label, value, sub) {
return h("div", { class: "card" }, h("div", { class: "label" }, label), h("div", { class: "value" }, value), sub ? h("div", { class: "sub" }, sub) : null);
}
// storageCard: the databases' size, and the fullest disk Jarl uses.
function storageCard(sg) {
const fullest = sg.disks.reduce((a, d) => (!a || used(d) > used(a) ? d : a), null);
const value = h("span", null, bytes(sg.total));
const sub = fullest ? [h("span", { class: used(fullest) > 0.9 ? "warn-text" : "" }, Math.round(used(fullest) * 100) + "% full"), " · " + fullest.path] : "databases";
return h("a", { class: "card link", href: "#storage" , onclick: (e) => { e.preventDefault(); document.getElementById("storage")?.scrollIntoView({ behavior: "smooth" }); } },
h("div", { class: "label" }, "Databases"), h("div", { class: "value" }, value), h("div", { class: "sub" }, sub));
}
// storagePanel shows each database's size and, per disk Jarl uses, a bar:
// the databases, everything else, and what is free.
function storagePanel(sg) {
const dbRows = sg.databases.map((d) => [h("dt", null, d.label),
h("dd", null, h("strong", null, bytes(d.bytes)), " ", h("span", { class: "mono small muted" }, d.path),
d.query_log_rows != null ? h("div", { class: "small muted" }, "the query log holds " + d.query_log_rows.toLocaleString() + " answers (stats.retention.queries)") : null)]);
return h("div", { class: "panel", id: "storage" },
h("h2", null, "Storage"),
h("dl", { class: "kv" }, dbRows, h("dt", null, "Together"), h("dd", null, h("strong", null, bytes(sg.total)))),
sg.disks.map((d) => {
const other = Math.max(0, d.total - d.free - d.databases);
const pctOf = (n) => (d.total ? (n / d.total) * 100 : 0) + "%";
const u = used(d);
return h("div", { class: "disk" },
h("div", { class: "disk-head" }, h("span", { class: "mono" }, d.path), h("span", { class: "muted small" }, " · " + d.holds.join(", ")),
h("span", { class: "grow" }), h("span", { class: u > 0.9 ? "warn-text" : "muted" }, bytes(d.free) + " free of " + bytes(d.total) + " (" + Math.round(u * 100) + "% used)")),
h("div", { class: "disk-bar", role: "img", "aria-label": Math.round(u * 100) + "% used, Jarl's databases " + bytes(d.databases) },
d.databases ? h("div", { class: "seg db", title: "Jarl's databases: " + bytes(d.databases), style: { width: pctOf(d.databases) } }) : null,
h("div", { class: "seg other" + (u > 0.9 ? " full" : ""), title: "Everything else: " + bytes(other), style: { width: pctOf(other) } })),
d.databases ? h("div", { class: "small muted" }, h("span", { class: "swatch db" }), " Jarl's databases " + bytes(d.databases), " ", h("span", { class: "swatch other" }), " everything else") : null);
}));
}
function used(d) { return d.total ? (d.total - d.free) / d.total : 0; }
function bytes(n) {
const units = ["B", "KB", "MB", "GB", "TB"];
let i = 0;
while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; }
return (n >= 100 || i === 0 ? Math.round(n) : n.toFixed(1)) + " " + units[i];
}
+66 -37
View File
@@ -26,7 +26,10 @@ export async function render(el) {
function statusPanel(s) {
const eps = (list) => list.map((e) => e.address + " port " + e.port).join(", ");
const rows = [
h("dt", null, "Certificate"), h("dd", null, h("a", { href: "#/certificates" }, s.certificate || "?"), s.hostname ? " (" + s.hostname + ")" : null),
h("dt", null, s.hostnames.length > 1 ? "Host names" : "Host name"), h("dd", { class: "mono" }, s.hostnames.join(", ") || "?"),
h("dt", null, "Certificate"), h("dd", null, h("a", { href: "#/certificates" }, s.certificate || "?")),
Object.keys(s.address_views || {}).length ? [h("dt", null, "Records"), h("dd", null, Object.entries(s.address_views).filter(([a]) => [...s.dot, ...s.doh].some((e) => e.address === a))
.map(([a, vs]) => h("div", null, h("span", { class: "mono" }, a), h("span", { class: "muted" }, " in " + vs.join(", ")))))] : null,
s.dot.length ? [h("dt", null, "DNS over TLS"), h("dd", { class: "mono" }, eps(s.dot))] : null,
s.doh.length ? [h("dt", null, "DNS over HTTPS"), h("dd", { class: "mono" }, eps(s.doh))] : null,
];
@@ -58,7 +61,8 @@ function statusPanel(s) {
}
function setupPanel(s) {
const host = input({ value: s.hostname || "", placeholder: "dns.adbraun.lol", class: "mono", style: { maxWidth: "320px" } });
const hostIn = input({ value: (s.hostnames || []).join(", "), placeholder: "dns.adbraun.lol", class: "mono", style: { maxWidth: "420px" } });
const hosts = () => hostIn.value.split(/[\s,]+/).map((x) => x.trim().toLowerCase().replace(/\.$/, "")).filter(Boolean);
const certSel = h("select");
const dot = checkbox("DNS over TLS", s.enabled ? s.dot.length > 0 : true);
const doh = checkbox("DNS over HTTPS", s.enabled ? s.doh.length > 0 : true);
@@ -69,40 +73,64 @@ function setupPanel(s) {
const portOut = h("div");
const err = h("div");
const progress = h("div", { class: "muted small" });
let views = s.views;
// Per address, the views where the host names get it: what their records
// say now (the setup's, or ones made by hand, which would otherwise be
// taken out), nothing else.
const viewsFor = new Map(s.addresses.map((a) => [a.address, new Set((s.address_views || {})[a.address] || [])]));
const isPrivate = new Map(s.addresses.map((a) => [a.address, a.private]));
// Addresses: one per row, nothing ticked (but the current setup's).
const addrRows = s.addresses.map((a) => pickRow(a.address, picked, [a.private ? h("span", { class: "muted" }, "private")
: a.address.includes(":") ? h("span", { class: "muted", title: "A global IPv6 address: reachable from the internet unless the firewall stops it" }, "global")
: badge("public", "warn")],
"Listen on " + a.address, () => checkPorts()));
// Views for the host name's records: the private ones that have its zone.
const viewPicked = new Set();
const viewBody = h("tbody");
const drawViews = (views) => {
viewPicked.clear();
mount(viewBody, views.map((v) => {
if (!v.public && v.has_zone) viewPicked.add(v.name);
return pickRow(v.name, viewPicked, [v.public ? badge("public", "warn") : h("span", { class: "muted" }, "private"), " ",
v.has_zone ? h("span", { class: "muted small" }, "has the zone") : h("span", { class: "muted small" }, "no zone for this name: skipped")], "Add the host name to view " + v.name);
}));
const viewCells = new Map();
const drawViewCell = (a) => {
const cell = viewCells.get(a);
if (!cell) return;
if (!picked.has(a)) return mount(cell);
const set = viewsFor.get(a);
const warn = isPrivate.get(a) && views.some((v) => v.public && set.has(v.name));
mount(cell, h("div", { class: "view-chips" }, h("span", { class: "muted small" }, "records in: "),
views.map((v) => {
const box = h("input", { type: "checkbox", checked: set.has(v.name), disabled: !v.has_zone && !set.has(v.name) });
box.addEventListener("change", () => { if (box.checked) set.add(v.name); else set.delete(v.name); drawViewCell(a); });
return h("label", { class: "chip" + (v.public ? " public" : ""), title: v.has_zone ? (v.public ? "A public view: the internet's clients get these records" : "") : "View " + v.name + " has no zone for the host names" }, box, " " + v.name, v.public ? h("span", { class: "muted small" }, " (public)") : null);
}),
!views.length ? h("span", { class: "muted small" }, "no views") : null),
warn ? h("div", { class: "msg warn small" }, a + " is a private address: clients of a public view can't reach it.") : null);
};
drawViews(s.views);
const addrRows = s.addresses.map((a) => {
const cell = h("td", { class: "addr-views" });
// The row toggles the address; clicks on its views shouldn't.
cell.addEventListener("click", (e) => e.stopPropagation());
viewCells.set(a.address, cell);
const tr = pickRow(a.address, picked, [a.private ? h("span", { class: "muted" }, "private")
: a.address.includes(":") ? h("span", { class: "muted", title: "A global IPv6 address: reachable from the internet unless the firewall stops it" }, "global")
: badge("public", "warn")],
"Listen on " + a.address, () => { drawViewCell(a.address); checkPorts(); });
tr.append(cell);
drawViewCell(a.address);
return tr;
});
const redrawViews = () => { for (const a of viewCells.keys()) drawViewCell(a); };
const fillCerts = () => {
const name = host.value.trim().toLowerCase().replace(/\.$/, "");
const fit = s.certificates.filter((c) => covers([...(c.names || []), ...(c.domains || [])], name));
const names = hosts();
const fit = s.certificates.filter((c) => names.length && names.every((n) => covers([...(c.names || []), ...(c.domains || [])], n)));
mount(certSel,
fit.map((c) => h("option", { value: c.name, selected: c.name === s.certificate }, c.name + " (" + (c.names || c.domains).join(", ") + (c.obtained ? "" : ", not obtained yet") + ")")),
name ? h("option", { value: "" }, "Obtain a new certificate for " + name) : h("option", { value: "" }, "Type the host name first"));
names.length ? h("option", { value: "" }, "Obtain a new certificate for " + names.join(", ")) : h("option", { value: "" }, "Type the host names first"));
};
fillCerts();
let viewTimer = null;
host.addEventListener("input", () => {
hostIn.addEventListener("input", () => {
fillCerts();
clearTimeout(viewTimer);
viewTimer = setTimeout(async () => {
const r = await get("/encrypted-dns", { hostname: host.value.trim() }).catch(() => null);
if (r) drawViews(r.views);
const r = await get("/encrypted-dns", { hostname: hosts().join(",") }).catch(() => null);
if (r) {
views = r.views;
// Records the new host names already have stay ticked.
for (const [a, vs] of Object.entries(r.address_views || {})) if (viewsFor.has(a)) vs.forEach((v) => viewsFor.get(a).add(v));
redrawViews();
}
}, 300);
});
@@ -129,23 +157,26 @@ function setupPanel(s) {
if (picked.size) checkPorts();
const body = (full) => {
const b = { hostname: host.value.trim(), addresses: [...picked], dot: dot.input.checked, doh: doh.input.checked, dot_port: Number(dotPort.value) || 853, doh_port: Number(dohPort.value) || 443 };
if (full) { b.certificate = certSel.value; b.views = [...viewPicked]; }
const b = { hostnames: hosts(), addresses: [...picked], dot: dot.input.checked, doh: doh.input.checked, dot_port: Number(dotPort.value) || 853, doh_port: Number(dohPort.value) || 443 };
if (full) {
b.certificate = certSel.value;
b.address_views = Object.fromEntries([...picked].map((a) => [a, [...(viewsFor.get(a) || [])]]));
}
return b;
};
const submit = async (btn) => {
mount(err);
const b = body(true);
if (!b.hostname) return mount(err, h("div", { class: "msg err" }, "Give the host name clients connect to."));
if (!b.hostnames.length) return mount(err, h("div", { class: "msg err" }, "Give the host name clients connect to."));
if (!b.addresses.length) return mount(err, h("div", { class: "msg err" }, "Choose at least one address to listen on."));
btn.disabled = true;
try {
if (!b.certificate) {
// Obtain one for the host name, for named, and wait for it.
b.certificate = b.hostname.split(".")[0].replace(/[^a-z0-9_-]/g, "-");
mount(progress, "Obtaining a certificate for " + b.hostname + " from the CA…");
await api("POST", "/certificates", { body: { name: b.certificate, domains: [b.hostname], uses: ["dns"] } });
// Obtain one for the host names, for named, and wait for it.
b.certificate = b.hostnames[0].split(".")[0].replace(/[^a-z0-9_-]/g, "-");
mount(progress, "Obtaining a certificate for " + b.hostnames.join(", ") + " from the CA…");
await api("POST", "/certificates", { body: { name: b.certificate, domains: b.hostnames, uses: ["dns"] } });
await waitObtained(b.certificate, progress);
} else {
const c = await get("/certificates/" + encodeURIComponent(b.certificate));
@@ -170,14 +201,12 @@ function setupPanel(s) {
return h("div", { class: "panel" },
h("h2", null, s.enabled ? "Change the setup" : "Set up"),
h("div", { class: "form-grid" },
field("Host name", host, "The name clients connect to. The certificate is for it, and it gets this server's addresses in the views below."),
field("Certificate", certSel, "One with this name, or a new one Jarl obtains and renews (docs/certificates.md)."),
field("Host names", hostIn, "The names clients connect to, separated by commas: dns.adbraun.lol, or several (dns.adbraun.lol, doh.adbraun.lol). The certificate must be for all of them."),
field("Certificate", certSel, "One for all the host names, or a new one Jarl obtains and renews (docs/certificates.md)."),
field("Listen on", h("div", null, h("div", { class: "table-wrap" }, h("table", { class: "pick-list" }, h("tbody", null, addrRows))), portOut),
"This server's addresses on your own networks. A public address works too, but then anyone who reaches it can use these listeners, as with port 53 there."),
"This server's addresses on your own networks. For each, tick the views where the host names get it (A or AAAA records, local to the view): a view's clients should only be given an address they can reach."),
field("Serve", h("div", { class: "enc-ports" }, h("div", null, dot, " on port ", dotPort), h("div", null, doh, " on port ", dohPort, h("span", { class: "muted small" }, " at /dns-query"))),
"Clients expect DoT on port 853. DoH can use any free port; 443 is the default, and needs nothing else listening there."),
field("Host name in views", h("div", { class: "table-wrap" }, h("table", { class: "pick-list" }, viewBody)),
"Where the host name gets the chosen addresses (A and AAAA), local to the view in a linked zone. Clients outside these views can't find it.")),
"Clients expect DoT on port 853. DoH can use any free port; 443 is the default, and needs nothing else listening there. Ports can be changed later: clients set up with another port need it changed too.")),
err, progress,
h("div", { class: "buttons" }, go));
}
+156
View File
@@ -0,0 +1,156 @@
// Exploring the query log (docs/stats.md#exploring): the answers of the
// last days, filtered by any mix of client, name, view, record type, result
// and blocking. Every value is a link that adds it to the filter, so a
// client leads to its names, a name to the clients asking for it, and so on.
import { h, mount, fmtTime, errorBox, badge, backLink } from "../ui.js";
import { get } from "../api.js";
import { lineChart, chartCard, legend, timeTable, barList, compact } from "../charts.js";
import { viewsOff } from "./stats.js";
const dims = [
["client", "Clients"], ["name", "Names"], ["view", "Views"],
["type", "Record types"], ["rcode", "Results"], ["blocked", "Blocked by"],
];
// What a filter is called, in the chips and the title.
const filterWords = { client: "client", name: "name", domain: "domain", view: "view", type: "type", rcode: "result", blocked: "blocked" };
const keys = Object.keys(filterWords);
const ranges = [["1h", 3600], ["6h", 6 * 3600], ["24h", 86400], ["3d", 3 * 86400], ["7d", 7 * 86400], ["30d", 30 * 86400]];
// exploreHref is the explore page for these filters (and range).
export function exploreHref(filters, since) {
const q = new URLSearchParams();
for (const k of keys) if (filters[k]) q.set(k, filters[k]);
if (since && since !== "24h") q.set("since", since);
return "#/stats/explore" + (q.toString() ? "?" + q : "");
}
const blockedLabel = (k) => (k === "rules" ? "your rules" : k === "yes" ? "anything" : k === "no" ? "nothing" : k);
export async function render(el, params, query) {
const f = {};
for (const k of keys) if (query.get(k)) f[k] = query.get(k);
const since = query.get("since") || "24h";
const with_ = (more) => exploreHref({ ...f, ...more }, since);
const without = (k) => { const g = { ...f }; delete g[k]; return exploreHref(g, since); };
// The Statistics page's view switches apply here too.
const off = viewsOff();
let views = "";
if (off.size) {
const all = (await get("/views").catch(() => [])).map((v) => v.name);
views = all.filter((v) => !off.has(v)).join(",");
}
let ex;
try {
ex = await get("/stats/explore", { ...f, since, views: views || undefined });
} catch (e) {
mount(el, head(f, without, since, null), e.code === "query_log_off"
? h("div", { class: "msg info" }, e.message) : errorBox(e));
return;
}
const parts = [head(f, without, since, ex)];
const maxSecs = retentionSecs(ex.retention);
parts.push(h("div", { class: "filter-row", role: "group", "aria-label": "Time range" },
h("span", { class: "muted" }, "Range"),
h("div", { class: "segmented" }, ranges.filter(([, s]) => s <= maxSecs).map(([k]) =>
h("a", { href: exploreHref(f, k), class: k === since ? "active" : "" }, k))),
h("span", { class: "muted small" }, " · the query log keeps " + ex.retention + (ex.oldest ? ", since " + fmtTime(ex.oldest) : "")),
off.size ? h("span", { class: "muted small" }, " · views switched off on the Statistics page are left out") : null));
const nx = (ex.by.rcode || []).find((e) => e.key === "NXDOMAIN");
parts.push(h("div", { class: "cards" },
tile("Answers", compact(ex.total), "from " + fmtTime(ex.from)),
tile("Blocked", compact(ex.blocked), ex.total ? pct(ex.blocked / ex.total) + " of them" : ""),
nx && !f.rcode ? tile("NXDOMAIN", compact(nx.count), pct(nx.count / Math.max(1, ex.total)) + " of them") : null,
!f.client && ex.by.client ? tile("Clients", (ex.by.client.length >= 15 ? "15+" : String(ex.by.client.length)), "asking") : null));
const times = ex.series.map((p) => p.t);
const series = [{ label: "Answers", values: ex.series.map((p) => p.count), slot: 1 }];
if (ex.blocked) series.push({ label: "Blocked", values: ex.series.map((p) => p.blocked), slot: 2 });
parts.push(chartCard(ex.blocked ? "Answers and blocked" : "Answers", "per " + stepWord(ex.step),
ex.total ? lineChart({ series, times, step: ex.step, area: true }) : h("div", { class: "empty" }, "No answers match."),
ex.total ? timeTable(series, times, ex.step) : null, { wide: true, legend: series.length > 1 ? legend(series) : null }));
// A breakdown for each dimension the filter leaves open: click a value
// to add it.
const lists = dims.filter(([d]) => ex.by[d] && !(d === "blocked" && !ex.blocked)).map(([d, title]) => {
const href = (k) => with_(d === "blocked" ? { blocked: k } : { [d]: k });
const label = d === "blocked" ? blockedLabel : d === "view" ? (k) => (k === "_default" ? "(no views)" : k) : (k) => k;
return h("div", { class: "card chart-card" }, h("h3", null, title),
barList(ex.by[d], { total: d === "blocked" ? ex.blocked : ex.total, href, label, empty: d === "blocked" ? "Nothing blocked." : "None." }));
});
if (lists.length) parts.push(h("div", { class: "chart-grid" }, lists));
parts.push(recentPanel(ex, with_));
mount(el, parts);
}
// head is the title and the filter chips, each with a ✕ to take it out.
function head(f, without, since, ex) {
const chips = keys.filter((k) => f[k]).map((k) => h("span", { class: "chip filter-chip" },
h("span", { class: "muted" }, filterWords[k] + " "),
h("strong", { class: "mono" }, k === "blocked" ? blockedLabel(f[k]) : f[k]),
k === "client" && ex && ex.client_name ? h("span", { class: "muted" }, " " + ex.client_name) : null,
" ", h("a", { href: without(k), title: "Take it out", class: "chip-x", "aria-label": "Remove the " + filterWords[k] + " filter" }, "✕")));
const extra = [];
if (f.name && !f.domain) extra.push(h("a", { href: exploreHref({ ...f, name: undefined, domain: f.name }, since) }, "this name and the names below it"));
if ((f.name || f.domain) && (f.blocked === "yes" || (ex && ex.blocked))) extra.push(h("a", { href: "#/blocking?check=" + encodeURIComponent(f.name || f.domain) }, "why is it blocked?"));
if (f.name || f.domain) extra.push(h("a", { href: "#/search?q=" + encodeURIComponent(f.name || f.domain) }, "find it in your zones"));
return h("div", { class: "page-head" },
h("div", { class: "grow" },
backLink("Statistics", "#/stats"),
h("h1", null, title(f, ex)),
h("div", { class: "muted" }, "Every answer in the query log that matches. Click a client, name, view, type or result to narrow it down; ✕ takes a filter out."),
chips.length ? h("div", { class: "chips filter-chips" }, chips,
h("a", { href: exploreHref({}, since), class: "small" }, "Clear all")) : null,
extra.length ? h("div", { class: "small" }, extra.map((a, i) => [i ? " · " : "", a])) : null));
}
function title(f, ex) {
const n = Object.keys(f).length;
if (!n) return "Explore the answers";
const who = f.client ? (ex && ex.client_name ? ex.client_name : f.client) : "";
if (f.client && n === 1) return "Client " + who;
const what = [f.rcode, f.type, f.blocked ? "blocked" + (f.blocked !== "yes" ? " by " + blockedLabel(f.blocked) : "") : "",
f.name || (f.domain ? f.domain + " and below" : "")].filter(Boolean).join(" ");
const parts = [what || "Answers"];
if (f.client) parts.push("for " + who);
if (f.view) parts.push("in view " + f.view);
return parts.join(" ");
}
// recentPanel lists the latest answers, every cell a link.
function recentPanel(ex, with_) {
if (!ex.recent.length) return null;
const a = (text, more, cls) => h("a", { href: with_(more), class: cls || "" }, text);
return h("div", { class: "panel" }, h("h2", null, "Latest answers"),
h("div", { class: "table-wrap" }, h("table", { class: "recent-queries" },
h("thead", null, h("tr", null, ["Time", "Client", "View", "Name", "Type", "Result"].map((t) => h("th", null, t)))),
h("tbody", null, ex.recent.map((r) => h("tr", null,
h("td", { class: "small muted nowrap" }, fmtTime(new Date(r.ts * 1000).toISOString())),
h("td", { class: "nowrap" }, a(r.client_name || r.client, { client: r.client }, "mono"), r.client_name ? h("span", { class: "muted small" }, " " + r.client) : null),
h("td", null, a(r.view === "_default" ? "–" : r.view, { view: r.view })),
h("td", { class: "oneline-cell" }, a(r.name, { name: r.name }, "mono")),
h("td", null, a(r.type, { type: r.type })),
h("td", null, r.blocked ? a(badge("blocked: " + blockedLabel(r.blocked), "warn"), { blocked: r.blocked }) : a(r.rcode, { rcode: r.rcode }, r.rcode === "NOERROR" ? "" : "warn-text"))))))),
ex.recent.length >= 100 ? h("div", { class: "muted small" }, "The latest 100: narrow the filter to see others.") : null);
}
function tile(label, value, sub) {
return h("div", { class: "card" }, h("div", { class: "label" }, label), h("div", { class: "value" }, value), h("div", { class: "sub" }, sub));
}
function pct(r) { return (r * 100).toFixed(r < 0.1 ? 1 : 0) + "%"; }
function stepWord(s) {
if (s % 86400 === 0) return s === 86400 ? "day" : s / 86400 + " days";
if (s % 3600 === 0) return s === 3600 ? "hour" : s / 3600 + " hours";
return s === 60 ? "minute" : s / 60 + " minutes";
}
function retentionSecs(r) {
const m = /^(\d+)([dhm])$/.exec(r || "");
if (!m) return 7 * 86400;
return Number(m[1]) * { d: 86400, h: 3600, m: 60 }[m[2]];
}
+23 -13
View File
@@ -2,6 +2,7 @@ import { h, mount, fmtTime, relTime, errorBox } from "../ui.js";
import { get } from "../api.js";
import { rerender } from "../app.js";
import { lineChart, chartCard, legend, timeTable, heatmap, heatmapTable, barList, compact, stepSeconds, endLabelWidth } from "../charts.js";
import { exploreHref } from "./explore.js";
const ranges = [["1h", "1 hour"], ["6h", "6 hours"], ["24h", "24 hours"], ["7d", "7 days"], ["30d", "30 days"], ["90d", "90 days"]];
const rangeSecs = { "1h": 3600, "6h": 6 * 3600, "24h": 86400, "7d": 7 * 86400, "30d": 30 * 86400, "90d": 90 * 86400 };
@@ -21,7 +22,7 @@ let generation = 0;
// The views shown are remembered per browser: the ones switched off, so a
// new view shows up switched on.
const offKey = "jarl.stats.views.off";
function viewsOff() {
export function viewsOff() {
try { return new Set(JSON.parse(localStorage.getItem(offKey) || "[]")); } catch { return new Set(); }
}
function saveViewsOff(set) {
@@ -114,17 +115,20 @@ export async function render(el, params, query) {
parts.push(chartCard("Busiest times", "average queries per hour by weekday and hour of day (local time), last " + ranges.find((r) => r[0] === heatSince)[1],
heatmap(heat, "queries"), heatmapTable(heat, "queries"), { wide: true }));
// Who asks for what.
// Who asks for what. With the query log, every value leads to the
// explore page: a client to what it asked, a name to who asked.
const ql = !!(status.retention || {}).queries;
const ex = (filters) => (ql ? (k) => exploreHref(filters(k), exploreSince(since)) : null);
if (status.query_log) {
const t = data.tops;
parts.push(h("div", { class: "chart-grid" },
listCard("Most queried names", "", barList(t.name, { href: (k) => "#/search?q=" + encodeURIComponent(k) })),
listCard("Busiest clients", "", barList(t.client)),
listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + "." })),
listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + "." })),
t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail)) : null,
t.blocked_client.length ? listCard("Clients with the most blocked queries", "often a smart TV, phone or app phoning home", barList(t.blocked_client)) : null,
t.blocked.length ? listCard("Most blocked names", "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: (k) => "#/blocking?check=" + encodeURIComponent(k) })) : null));
listCard("Most queried names", ql ? "click one to see who asks for it" : "", barList(t.name, { href: ex((k) => ({ name: k })) || ((k) => "#/search?q=" + encodeURIComponent(k)) })),
listCard("Busiest clients", ql ? "click one to see what it asks" : "", barList(t.client, { href: ex((k) => ({ client: k })) })),
listCard("Names that don't exist", "NXDOMAIN answers: typos, removed hosts, misconfigured clients, malware", barList(t.nxdomain, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ name: k, rcode: "NXDOMAIN" })) })),
listCard("Refused clients", "clients asking for something they may not have (often internet scanners)", barList(t.refused, { empty: "None " + rangeLabel + ".", href: ex((k) => ({ client: k, rcode: "REFUSED" })) })),
t.servfail.length ? listCard("Names that failed to resolve", "SERVFAIL answers", barList(t.servfail, { href: ex((k) => ({ name: k, rcode: "SERVFAIL" })) })) : null,
t.blocked_client.length ? listCard("Clients with the most blocked queries", "often a smart TV, phone or app phoning home", barList(t.blocked_client, { href: ex((k) => ({ client: k, blocked: "yes" })) })) : null,
t.blocked.length ? listCard("Most blocked names", ql ? "click a name to see who asks for it, and why it is blocked" : "answered \"doesn't exist\" by blocking; click a name to see why, or to allow it", barList(t.blocked, { href: ex((k) => ({ name: k, blocked: "yes" })) || ((k) => "#/blocking?check=" + encodeURIComponent(k)) })) : null));
} else {
parts.push(h("div", { class: "msg info" }, "Top names and clients need the response log: set stats.query_log: true in /etc/jarl/jarl.yaml and restart jarl."));
}
@@ -133,13 +137,13 @@ export async function render(el, params, query) {
const b = data.breakdowns;
const rtt = rttOrder.map((k) => b.rtt.find((e) => e.key === k)).filter(Boolean);
parts.push(h("div", { class: "chart-grid" },
listCard("Record types", "", barList(b.qtype, { total: s.queries })),
listCard("Result codes", "", barList(b.rcode, { total: s.queries })),
b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel })) : null,
listCard("Record types", "", barList(b.qtype, { total: s.queries, href: ex((k) => ({ type: k })) })),
listCard("Result codes", ql ? "click one to see who gets it, for what" : "", barList(b.rcode, { total: s.queries, href: ex((k) => ({ rcode: k })) })),
b.view.length > 1 || (b.view.length === 1 && b.view[0].key !== "_default") ? listCard("Views", "", barList(b.view, { label: viewLabel, href: ex((k) => ({ view: k })) })) : null,
b.zone.length ? listCard("Zones", "queries per local zone; other = names in no local zone (recursion)", barList(b.zone, { label: (k) => (k === "-" ? "other" : k), href: (k) => (k === "-" ? null : "#/search?q=" + encodeURIComponent(k)) })) : null,
b.answer.length ? listCard("Answers", "authoritative (your zones) or recursive (from the cache or the internet)", barList(b.answer)) : null,
rtt.length ? listCard("Upstream response time", "how long other servers took to answer named's recursive queries", barList(rtt)) : null,
b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: () => "#/blocking" })) : null));
b.blocklist.length ? listCard("Blocked by", "which of your rules and blocklists blocked queries", barList(b.blocklist, { label: (k) => (k === "rules" ? "your rules" : k), href: ex((k) => ({ blocked: k })) || (() => "#/blocking") })) : null));
parts.push(...serverSection(data.server, complete(data.gauges), rangeLabel));
el.classList.remove("refreshing");
@@ -302,3 +306,9 @@ function disabledPanel() {
h("p", { class: "muted" }, "Jarl then makes named publish its counters on a local statistics channel and log responses to /var/log/named, and keeps aggregates " +
"(never individual queries) in a separate database, by default for 7 days per minute, 90 days per hour and 2 years per day. See docs/stats.md."));
}
// exploreSince is the Statistics range for the explore page: the query log
// keeps days, not months.
function exploreSince(since) {
return rangeSecs[since] > 7 * 86400 ? "7d" : since;
}
+23
View File
@@ -590,3 +590,26 @@ pre.wrap { white-space: pre-wrap; word-break: break-all; }
.size-chips button.active { background: var(--accent); border-color: var(--accent); color: var(--accent-text); }
.cands { display: flex; flex-direction: column; gap: 3px; max-height: 260px; overflow-y: auto; }
.cands button.cand { flex-shrink: 0; text-align: left; font-weight: normal; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
/* Encrypted DNS: per listen address, the views that get its records. */
td.addr-views { width: 100%; }
.view-chips { display: flex; flex-wrap: wrap; gap: 4px 10px; align-items: center; }
.view-chips label.chip { white-space: nowrap; font-size: 0.9em; }
/* Explore: the filter chips with a ✕ each, and the latest answers. */
.filter-chips { margin-top: 8px; display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
.chip.filter-chip { display: inline-flex; gap: 2px; align-items: center; padding: 2px 6px 2px 10px; border-radius: 999px; background: var(--info-soft); border: 1px solid var(--border); font-size: 13px; }
.chip.filter-chip a.chip-x { text-decoration: none; color: var(--muted); padding: 0 4px; }
.chip.filter-chip a.chip-x:hover { color: var(--err, #c00); }
table.recent-queries td.oneline-cell { max-width: 40ch; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
/* Dashboard storage: a bar per disk. */
.disk { margin-top: 14px; }
.disk-head { display: flex; gap: 6px; align-items: baseline; flex-wrap: wrap; margin-bottom: 4px; }
.disk-head .grow { flex: 1; }
.disk-bar { display: flex; height: 10px; border-radius: 5px; background: var(--info-soft); overflow: hidden; border: 1px solid var(--border); }
.disk-bar .seg.db { background: var(--accent); }
.disk-bar .seg.other { background: var(--muted); opacity: 0.45; }
.disk-bar .seg.other.full { background: var(--warn); opacity: 0.8; }
.swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; vertical-align: -1px; }
.swatch.db { background: var(--accent); }
.swatch.other { background: var(--muted); opacity: 0.45; }
+7
View File
@@ -901,6 +901,13 @@ if [ -n "$PEBBLE" ]; then
sleep 16 # /metrics caches Jarl's state for 15 seconds
run "health metric" 0 curl -sf http://127.0.0.1:$HTTP/metrics
contains "^jarl_encrypted_dns_up{kind=\"dot\",address=\"127.0.0.1\",port=\"$EDOT\"} 1$"
run "cert show: what named uses it for" 0 $J cert show ns1
contains "^named: *DNS over TLS on 127.0.0.1 port $EDOT"
contains "^named: *DNS over HTTPS on 127.0.0.1 port $EDOH"
run "per address views: 127.0.0.1 only in internal" 0 $J encrypted-dns setup --hostname ns1.adbraun.test --address 127.0.0.1=internal --dot-port $EDOT --doh-port $EDOH
run "status shows where the records are" 0 $J encrypted-dns
contains "^Records: *127.0.0.1 in internal$"
contains "^Host names: *ns1.adbraun.test$"
run "a certificate named uses can't be deleted (exit 6)" 6 $J cert delete ns1 --yes
contains "take it out first"
run "encrypted-dns disable" 0 $J encrypted-dns disable