Encrypted DNS health, comparing a zone's views, --local everywhere

jarld checks the DoT and DoH listeners every 10 minutes as a client would (page, dashboard, jarl encrypted-dns health, jarl_encrypted_dns_up). The Compare views tab and jarl zone diff show what differs between a zone's views, with Use this version. --local works on unlinked zones. A changed certificates.dir is followed by encrypted DNS's tls statements. Sharing an already shared version replaces other views' overrides.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/encdns/encdns.go

@@ -272,9 +272,7 @@ func Plan(st *model.State, s Setup, certDir string) ([]engine.Request, []string,
 		return nil, nil, fmt.Errorf("choose at least one address to listen on")
 	}
 	tlsName := Prefix + s.Certificate
-	dir := filepath.Join(certDir, s.Certificate)
-	tlsValue := fmt.Sprintf(`%s { cert-file "%s"; key-file "%s"; protocols { TLSv1.2; TLSv1.3; }; }`,
-		tlsName, filepath.Join(dir, "fullchain.pem"), filepath.Join(dir, "privkey.pem"))
+	tlsValue := TLSValue(s.Certificate, certDir)
 
 	var reqs []engine.Request
 	// Take out the earlier setup's listeners and tls statements.
@@ -393,3 +391,11 @@ func Private(a netip.Addr) bool {
 	a = a.Unmap()
 	return a.IsPrivate() || a.IsLoopback() || a.IsLinkLocalUnicast() || netip.MustParsePrefix("100.64.0.0/10").Contains(a)
 }
+
+// TLSValue is the tls statement the setup writes for a certificate whose
+// files are in certDir/NAME.
+func TLSValue(cert, certDir string) string {
+	dir := filepath.Join(certDir, cert)
+	return fmt.Sprintf(`%s { cert-file "%s"; key-file "%s"; protocols { TLSv1.2; TLSv1.3; }; }`,
+		Prefix+cert, filepath.Join(dir, "fullchain.pem"), filepath.Join(dir, "privkey.pem"))
+}

internal/encdns/health.go

@@ -0,0 +1,208 @@
+package encdns
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+// Probe is the outcome of checking one DoT or DoH listener the way a
+// client would: connect, check the certificate, ask a question.
+type Probe struct {
+	Kind    string `json:"kind"` // dot or doh
+	Address string `json:"address"`
+	Port    int    `json:"port"`
+	// OK: it answered over TLS with a certificate for the host name that
+	// is current. Problem says what is wrong when not; Warning notes
+	// something that doesn't stop clients from working everywhere.
+	OK      bool   `json:"ok"`
+	Problem string `json:"problem,omitempty"`
+	Warning string `json:"warning,omitempty"`
+	// What the certificate named served says.
+	CertNames    []string   `json:"cert_names,omitempty"`
+	CertNotAfter *time.Time `json:"cert_not_after,omitempty"`
+	CertSerial   string     `json:"cert_serial,omitempty"`
+	// Trusted: it chains to a CA this server trusts (not so for a
+	// staging or test CA).
+	Trusted bool   `json:"trusted"`
+	RCode   string `json:"rcode,omitempty"`
+	Millis  int64  `json:"ms"`
+}
+
+// Health is the last check of every listener.
+type Health struct {
+	Checked time.Time `json:"checked"`
+	Probes  []Probe   `json:"probes"`
+}
+
+// OK reports whether every listener works.
+func (h *Health) OK() bool {
+	for _, p := range h.Probes {
+		if !p.OK {
+			return false
+		}
+	}
+	return true
+}
+
+// Check probes every DoT and DoH listener of the setup. hostname is what
+// clients ask for, serial the certificate Jarl last wrote for named ("" if
+// unknown), roots the CAs to trust (nil: the system's).
+func Check(ctx context.Context, s Status, hostname, serial string, roots *x509.CertPool) *Health {
+	h := &Health{Checked: time.Now(), Probes: []Probe{}}
+	for _, e := range s.DoT {
+		h.Probes = append(h.Probes, probe(ctx, "dot", e, hostname, serial, roots))
+	}
+	for _, e := range s.DoH {
+		h.Probes = append(h.Probes, probe(ctx, "doh", e, hostname, serial, roots))
+	}
+	return h
+}
+
+const probeTimeout = 5 * time.Second
+
+func probe(ctx context.Context, kind string, e Endpoint, hostname, serial string, roots *x509.CertPool) Probe {
+	p := Probe{Kind: kind, Address: e.Address, Port: e.Port}
+	addr := net.JoinHostPort(e.Address, strconv.Itoa(e.Port))
+	ctx, cancel := context.WithTimeout(ctx, probeTimeout)
+	defer cancel()
+	start := time.Now()
+	q := new(dns.Msg)
+	q.SetQuestion(dns.Fqdn(hostname), dns.TypeA)
+	q.RecursionDesired = true
+	var state tls.ConnectionState
+	var answer *dns.Msg
+	var err error
+	cfg := &tls.Config{ServerName: hostname, InsecureSkipVerify: true, MinVersion: tls.VersionTLS12} // checked below, to say what is wrong
+	if kind == "dot" {
+		cfg.NextProtos = []string{"dot"}
+		state, answer, err = askDoT(ctx, addr, cfg, q)
+	} else {
+		state, answer, err = askDoH(ctx, addr, hostname, e.Port, cfg, q)
+	}
+	p.Millis = time.Since(start).Milliseconds()
+	if len(state.PeerCertificates) > 0 {
+		leaf := state.PeerCertificates[0]
+		p.CertNames, p.CertSerial = leaf.DNSNames, fmt.Sprintf("%x", leaf.SerialNumber)
+		na := leaf.NotAfter
+		p.CertNotAfter = &na
+		inter := x509.NewCertPool()
+		for _, c := range state.PeerCertificates[1:] {
+			inter.AddCert(c)
+		}
+		_, verr := leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: inter, DNSName: hostname})
+		p.Trusted = verr == nil
+		switch {
+		case leaf.VerifyHostname(hostname) != nil:
+			p.Problem = fmt.Sprintf("named serves a certificate for %s, not %s: clients reject it", strings.Join(leaf.DNSNames, ", "), hostname)
+		case time.Now().After(leaf.NotAfter):
+			p.Problem = "named serves an expired certificate (until " + leaf.NotAfter.Format(time.DateOnly) + ")"
+		case serial != "" && p.CertSerial != serial:
+			p.Problem = "named still serves an older certificate (serial " + p.CertSerial + ", Jarl has " + serial + "): it hasn't reread the files (rndc reconfig)"
+		case !p.Trusted:
+			p.Warning = "the certificate isn't from a CA this server trusts (a staging or test CA?): clients reject it unless they trust that CA"
+		}
+	}
+	switch {
+	case err != nil:
+		p.Problem = describe(err, e)
+	case p.Problem != "":
+	case answer == nil:
+		p.Problem = "no DNS answer"
+	default:
+		p.RCode = dns.RcodeToString[answer.Rcode]
+		p.OK = true
+		if answer.Rcode == dns.RcodeRefused {
+			p.Warning = strings.TrimPrefix(p.Warning+"; ", "; ") + "it answers, but REFUSED: this server's own address gets a view that doesn't answer for " + hostname
+		}
+	}
+	return p
+}
+
+func askDoT(ctx context.Context, addr string, cfg *tls.Config, q *dns.Msg) (tls.ConnectionState, *dns.Msg, error) {
+	d := &tls.Dialer{NetDialer: &net.Dialer{Timeout: probeTimeout}, Config: cfg}
+	conn, err := d.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return tls.ConnectionState{}, nil, err
+	}
+	defer conn.Close()
+	tc := conn.(*tls.Conn)
+	state := tc.ConnectionState()
+	conn.SetDeadline(time.Now().Add(probeTimeout))
+	co := &dns.Conn{Conn: conn}
+	if err := co.WriteMsg(q); err != nil {
+		return state, nil, err
+	}
+	r, err := co.ReadMsg()
+	return state, r, err
+}
+
+func askDoH(ctx context.Context, addr, hostname string, port int, cfg *tls.Config, q *dns.Msg) (tls.ConnectionState, *dns.Msg, error) {
+	cfg = cfg.Clone()
+	cfg.NextProtos = []string{"h2", "http/1.1"}
+	tr := &http.Transport{
+		TLSClientConfig:   cfg,
+		ForceAttemptHTTP2: true,
+		// Whatever the host name resolves to here: the listener's address.
+		DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
+			return (&net.Dialer{Timeout: probeTimeout}).DialContext(ctx, network, addr)
+		},
+	}
+	defer tr.CloseIdleConnections()
+	wire, err := q.Pack()
+	if err != nil {
+		return tls.ConnectionState{}, nil, err
+	}
+	url := "https://" + net.JoinHostPort(hostname, strconv.Itoa(port)) + "/dns-query"
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(wire))
+	if err != nil {
+		return tls.ConnectionState{}, nil, err
+	}
+	req.Header.Set("Content-Type", "application/dns-message")
+	req.Header.Set("Accept", "application/dns-message")
+	resp, err := (&http.Client{Transport: tr, Timeout: probeTimeout}).Do(req)
+	if err != nil {
+		return tls.ConnectionState{}, nil, err
+	}
+	defer resp.Body.Close()
+	var state tls.ConnectionState
+	if resp.TLS != nil {
+		state = *resp.TLS
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+	if err != nil {
+		return state, nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return state, nil, fmt.Errorf("HTTP %d at /dns-query", resp.StatusCode)
+	}
+	r := new(dns.Msg)
+	if err := r.Unpack(body); err != nil {
+		return state, nil, fmt.Errorf("not a DNS answer: %w", err)
+	}
+	return state, r, nil
+}
+
+// describe says what a failed probe means.
+func describe(err error, e Endpoint) string {
+	var ne net.Error
+	switch {
+	case errors.Is(err, syscall.ECONNREFUSED):
+		return fmt.Sprintf("nothing listens on %s port %d: named may not have been able to bind it (look in named's log; ports below 1024 need named to start as root)", e.Address, e.Port)
+	case errors.As(err, &ne) && ne.Timeout(), errors.Is(err, context.DeadlineExceeded):
+		return fmt.Sprintf("no answer from %s port %d within %s (a firewall on the server?)", e.Address, e.Port, probeTimeout)
+	}
+	return err.Error()
+}

internal/encdns/health_test.go

@@ -0,0 +1,115 @@
+package encdns
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"io"
+	"math/big"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+func testCert(t *testing.T, names ...string) (tls.Certificate, *x509.Certificate) {
+	key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	tmpl := &x509.Certificate{SerialNumber: big.NewInt(0xabc123), Subject: pkix.Name{CommonName: names[0]}, DNSNames: names,
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf, _ := x509.ParseCertificate(der)
+	return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, leaf
+}
+
+// answer answers every question with 192.0.2.53.
+var answer = dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	m.Answer = append(m.Answer, &dns.A{Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("192.0.2.53")})
+	w.WriteMsg(m)
+})
+
+func listeners(t *testing.T, cert tls.Certificate) (dot, doh Endpoint) {
+	cfg := &tls.Config{Certificates: []tls.Certificate{cert}}
+	ln, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := &dns.Server{Listener: ln, Net: "tcp-tls", Handler: answer}
+	go srv.ActivateAndServe()
+	t.Cleanup(func() { srv.Shutdown() })
+	hl, err := tls.Listen("tcp", "127.0.0.1:0", cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hs := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/dns-query" {
+			http.NotFound(w, r)
+			return
+		}
+		b, _ := io.ReadAll(r.Body)
+		q := new(dns.Msg)
+		if q.Unpack(b) != nil {
+			http.Error(w, "bad", 400)
+			return
+		}
+		m := new(dns.Msg)
+		m.SetReply(q)
+		m.Rcode = dns.RcodeSuccess
+		out, _ := m.Pack()
+		w.Header().Set("Content-Type", "application/dns-message")
+		w.Write(out)
+	})}
+	go hs.Serve(hl)
+	t.Cleanup(func() { hs.Close() })
+	port := func(l net.Listener) int { return l.Addr().(*net.TCPAddr).Port }
+	return Endpoint{"127.0.0.1", port(ln)}, Endpoint{"127.0.0.1", port(hl)}
+}
+
+func TestHealth(t *testing.T) {
+	ctx := context.Background()
+	cert, leaf := testCert(t, "dns.example.com")
+	dot, doh := listeners(t, cert)
+	roots := x509.NewCertPool()
+	roots.AddCert(leaf)
+	st := Status{Enabled: true, DoT: []Endpoint{dot}, DoH: []Endpoint{doh}}
+
+	h := Check(ctx, st, "dns.example.com", "abc123", roots)
+	if !h.OK() || len(h.Probes) != 2 {
+		t.Fatalf("working setup: %+v", h.Probes)
+	}
+	for _, p := range h.Probes {
+		if !p.Trusted || p.RCode != "NOERROR" || p.Warning != "" || p.CertSerial != "abc123" {
+			t.Errorf("%s: %+v", p.Kind, p)
+		}
+	}
+	// named hasn't reread a renewed certificate.
+	if h := Check(ctx, st, "dns.example.com", "def456", roots); h.OK() || !strings.Contains(h.Probes[0].Problem, "older certificate") {
+		t.Errorf("stale: %+v", h.Probes[0])
+	}
+	// The wrong name.
+	if h := Check(ctx, st, "other.example.com", "", roots); h.OK() || !strings.Contains(h.Probes[0].Problem, "not other.example.com") {
+		t.Errorf("wrong name: %+v", h.Probes[0])
+	}
+	// A CA this server doesn't trust: works, with a warning.
+	if h := Check(ctx, st, "dns.example.com", "abc123", x509.NewCertPool()); !h.OK() || !strings.Contains(h.Probes[1].Warning, "staging") {
+		t.Errorf("untrusted: %+v", h.Probes[1])
+	}
+	// Nothing listening.
+	l, _ := net.Listen("tcp", "127.0.0.1:0")
+	gone := l.Addr().(*net.TCPAddr).Port
+	l.Close()
+	if h := Check(ctx, Status{Enabled: true, DoT: []Endpoint{{"127.0.0.1", gone}}}, "dns.example.com", "", roots); h.OK() || !strings.Contains(h.Probes[0].Problem, "nothing listens") {
+		t.Errorf("nothing listening: %+v", h.Probes[0])
+	}
+}