The web UI's certificate from Let's Encrypt, through Jarl

tls.acme names the certificate and jarl: the Jarl server and a token that
may only write the challenge record. Steward gets the certificate with
DNS-01 (lego), writing the record through Jarl's API into every view that
has the zone, renews it with a third of its lifetime left and swaps it
in without a restart. A check runs before every order: Jarl, the zone,
the token's permission, a test record, the zone's name servers, public
resolvers, clean-up; each step says what to fix. Staging by default.

With a certificate, each listener serves HTTPS and plain HTTP on one port
(redirecting plain HTTP, except from the server itself).

steward cert / cert check / cert renew, Server -> Certificate in the web
UI, GET /certificate, POST /certificate/check|renew. docs/certificates.md
and docs/jarl.md (the Jarl user and the token's grant).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/certs/check.go

@@ -0,0 +1,379 @@
+package certs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"steward/internal/api"
+	"steward/internal/auth"
+	"steward/internal/jarl"
+)
+
+// Step and Check are api.CertStep and api.CertCheck.
+type (
+	Step  = api.CertStep
+	Check = api.CertCheck
+)
+
+// note records a query and its answer, replacing an earlier answer to
+// the same question (the checks ask again until the record shows up).
+func note(s *Step, q string) {
+	cmd, _, _ := strings.Cut(q, "  → ")
+	for i, old := range s.Queries {
+		if prev, _, _ := strings.Cut(old, "  → "); prev == cmd {
+			s.Queries[i] = q
+			return
+		}
+	}
+	s.Queries = append(s.Queries, q)
+}
+
+func add(c *Check, name, status, format string, args ...any) *Step {
+	c.Steps = append(c.Steps, Step{Name: name, Status: status, Detail: fmt.Sprintf(format, args...)})
+	return &c.Steps[len(c.Steps)-1]
+}
+
+// How long the check waits for the record to show up, and the name
+// servers' port (tests change them).
+var (
+	propagationWait = 90 * time.Second
+	pollEvery       = 2 * time.Second
+	nsPort          = "53"
+)
+
+// Preflight checks, without the CA, that a certificate can be obtained:
+// Jarl answers and knows the token, a zone holds the challenge name, the
+// token may write it, a test record can be written, the zone's name
+// servers and public resolvers serve it, and it can be removed again.
+func (m *Manager) Preflight(ctx context.Context) *Check {
+	c := &Check{Time: time.Now().UTC(), Domain: m.cfg.ACME.Domain}
+	defer func() {
+		c.OK = true
+		for _, s := range c.Steps {
+			if s.Status == "fail" {
+				c.OK = false
+			}
+		}
+		m.mu.Lock()
+		m.lastCheck = c
+		m.mu.Unlock()
+	}()
+	name := jarl.ChallengeName(m.cfg.ACME.Domain)
+	// A CNAME at the challenge name sends the challenge elsewhere (into a
+	// zone kept for challenges); the CA and lego follow it, so the check
+	// does too.
+	if target := m.cname(ctx, name); target != "" {
+		name = target
+	}
+	shown := strings.TrimSuffix(name, ".")
+
+	who, err := m.jarl.Whoami(ctx)
+	if err != nil {
+		add(c, "Jarl", "fail", "%s", explainJarl(err))
+		return c
+	}
+	add(c, "Jarl", "ok", "the token is %s's, on Jarl %s", who.User.Username, who.Build.Label)
+
+	all, err := m.jarl.Zones(ctx)
+	if err != nil {
+		add(c, "Zone", "fail", "listing the zones: %s", explainJarl(err))
+		return c
+	}
+	zones := jarl.ZonesFor(name, all, "")
+	if len(zones) == 0 {
+		add(c, "Zone", "fail", "no primary zone the token can see holds %s: add the zone to Jarl, or give the token's user access to it (zones=...)", shown)
+		return c
+	}
+	zone := strings.TrimSuffix(zones[0].Name, ".")
+	// Every view with the zone gets the record (linked zones through
+	// their link), so name them all.
+	var inViews []jarl.Zone
+	for _, z := range all {
+		if strings.EqualFold(z.Name, zones[0].Name) {
+			inViews = append(inViews, z)
+		}
+	}
+	add(c, "Zone", "ok", "%s%s", zone, viewsText(inViews))
+
+	value := "steward-check-" + auth.RandomString(8)
+	if err := m.jarl.SetTXT(ctx, name, value, zones, true, true, "Steward's certificate check"); err != nil {
+		add(c, "Permission", "fail", "%s", explainWrite(err, zone, shown))
+		return c
+	}
+	add(c, "Permission", "ok", "the token may write TXT records at %s", shown)
+
+	if err := m.jarl.SetTXT(ctx, name, value, zones, true, false, "Steward's certificate check"); err != nil {
+		add(c, "Write", "fail", "writing a test record: %s", explainWrite(err, zone, shown))
+		return c
+	}
+	add(c, "Write", "ok", "wrote TXT %q at %s; Jarl has it in named", value, shown)
+	defer func() {
+		clean, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+		defer cancel()
+		if err := m.jarl.SetTXT(clean, name, value, zones, false, false, "Steward's certificate check"); err != nil {
+			add(c, "Clean up", "fail", "removing the test record: %s (remove it in Jarl by hand)", explainJarl(err))
+			return
+		}
+		add(c, "Clean up", "ok", "removed the test record")
+	}()
+
+	authOK := true
+	if m.cfg.ACME.CheckPropagation {
+		authOK = m.checkNameServers(ctx, c, zone, name, value)
+	} else {
+		add(c, "Name servers", "skip", "tls.acme.check_propagation is off")
+	}
+	m.checkResolvers(ctx, c, name, value, authOK && m.cfg.ACME.CheckPropagation)
+	return c
+}
+
+func viewsText(zs []jarl.Zone) string {
+	var views []string
+	for _, z := range zs {
+		if z.View != "" {
+			views = append(views, z.View)
+		}
+	}
+	switch len(views) {
+	case 0:
+		return ""
+	case 1:
+		return " in view " + views[0]
+	}
+	return " in views " + strings.Join(views, ", ")
+}
+
+func explainJarl(err error) string {
+	var je *jarl.Error
+	if errors.As(err, &je) && je.Status == 401 {
+		return "Jarl refused the token (" + je.Message + "): check jarl.token or jarl.token_file"
+	}
+	return err.Error()
+}
+
+func explainWrite(err error, zone, name string) string {
+	var je *jarl.Error
+	if errors.As(err, &je) && je.Status == 403 {
+		return fmt.Sprintf("the token may not write the challenge record (%s); give it the grant: records:write zones=%s names=%s types=TXT", je.Message, zone, name)
+	}
+	return explainJarl(err)
+}
+
+// checkNameServers asks each of the zone's public name servers directly
+// (as the CA's resolvers will) until all serve the value.
+func (m *Manager) checkNameServers(ctx context.Context, c *Check, zone, name, value string) bool {
+	st := add(c, "Name servers", "fail", "")
+	ns, q, err := m.lookupNS(ctx, zone)
+	st.Queries = append(st.Queries, q...)
+	if err != nil || len(ns) == 0 {
+		st.Detail = fmt.Sprintf("the public resolvers know no name servers for %s, so the CA can't see it: is the zone delegated to your name servers? (%v)", zone, err)
+		return false
+	}
+	type server struct {
+		host  string
+		addrs []string
+	}
+	var servers []server
+	for _, h := range ns {
+		addrs, q := m.lookupAddrs(ctx, h)
+		st.Queries = append(st.Queries, q...)
+		if len(addrs) == 0 {
+			st.Detail = fmt.Sprintf("name server %s has no address the public resolvers know", strings.TrimSuffix(h, "."))
+			return false
+		}
+		servers = append(servers, server{h, addrs})
+	}
+	// A server passes when any of its addresses serves the value (this
+	// machine may not reach its IPv6 address, for one).
+	deadline := time.Now().Add(propagationWait)
+	missing := servers
+	for {
+		var still []server
+		for _, s := range missing {
+			found := false
+			for _, a := range s.addrs {
+				ok, q := m.hasTXT(ctx, net.JoinHostPort(a, nsPort), name, value, false)
+				note(st, q)
+				if ok {
+					found = true
+					break
+				}
+			}
+			if !found {
+				still = append(still, s)
+			}
+		}
+		missing = still
+		if len(missing) == 0 {
+			st.Status, st.Detail = "ok", fmt.Sprintf("all %d name servers of %s serve it", len(servers), zone)
+			return true
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			var names []string
+			for _, s := range missing {
+				names = append(names, strings.TrimSuffix(s.host, ".")+" ("+strings.Join(s.addrs, ", ")+")")
+			}
+			st.Detail = fmt.Sprintf("after %s, %s still don't serve it: a secondary that doesn't get the zone's updates (check NOTIFY and transfers), or an address that doesn't reach your name server", propagationWait, strings.Join(names, ", "))
+			return false
+		}
+		sleep(ctx, pollEvery)
+	}
+}
+
+// checkResolvers asks the public resolvers until each returns the value.
+// When the name servers already serve it, a resolver that doesn't is only
+// a warning: it may be holding an earlier answer.
+func (m *Manager) checkResolvers(ctx context.Context, c *Check, name, value string, soft bool) {
+	st := add(c, "Public resolvers", "fail", "")
+	if len(m.cfg.ACME.Resolvers) == 0 {
+		st.Status, st.Detail = "skip", "no tls.acme.resolvers"
+		return
+	}
+	deadline := time.Now().Add(propagationWait)
+	missing := append([]string(nil), m.cfg.ACME.Resolvers...)
+	for {
+		var still []string
+		for _, r := range missing {
+			ok, q := m.hasTXT(ctx, r, name, value, true)
+			note(st, q)
+			if !ok {
+				still = append(still, r)
+			}
+		}
+		missing = still
+		if len(missing) == 0 {
+			st.Status, st.Detail = "ok", "every resolver sees it"
+			return
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			st.Detail = fmt.Sprintf("after %s, %s don't see it", propagationWait, strings.Join(missing, ", "))
+			if soft {
+				st.Status = "warn"
+				st.Detail += "; the name servers do, so they are probably holding an earlier answer"
+			}
+			return
+		}
+		sleep(ctx, pollEvery)
+	}
+}
+
+func sleep(ctx context.Context, d time.Duration) {
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// exchange asks server (HOST:PORT) one question.
+func exchange(ctx context.Context, server, name string, qtype uint16, recurse bool) (*dns.Msg, string, error) {
+	if _, _, err := net.SplitHostPort(server); err != nil {
+		server = net.JoinHostPort(server, "53")
+	}
+	m := new(dns.Msg)
+	m.SetQuestion(dns.Fqdn(name), qtype)
+	m.RecursionDesired = recurse
+	host, port, _ := net.SplitHostPort(server)
+	q := fmt.Sprintf("dig @%s -p %s %s %s", host, port, dns.TypeToString[qtype], strings.TrimSuffix(name, "."))
+	if !recurse {
+		q += " +norec"
+	}
+	c, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	cl := &dns.Client{Timeout: 5 * time.Second}
+	r, _, err := cl.ExchangeContext(c, m, server)
+	if err == nil && r.Truncated {
+		cl.Net = "tcp"
+		r, _, err = cl.ExchangeContext(c, m, server)
+	}
+	return r, q, err
+}
+
+func (m *Manager) hasTXT(ctx context.Context, server, name, value string, recurse bool) (bool, string) {
+	r, q, err := exchange(ctx, server, name, dns.TypeTXT, recurse)
+	if err != nil {
+		return false, q + "  → " + err.Error()
+	}
+	for _, rr := range r.Answer {
+		if t, ok := rr.(*dns.TXT); ok && strings.Join(t.Txt, "") == value {
+			return true, q + "  → found"
+		}
+	}
+	return false, q + "  → " + dns.RcodeToString[r.Rcode] + ", not there (yet)"
+}
+
+// cname returns the target of a CNAME at name ("" when there is none).
+func (m *Manager) cname(ctx context.Context, name string) string {
+	for _, res := range m.cfg.ACME.Resolvers {
+		r, _, err := exchange(ctx, res, name, dns.TypeCNAME, true)
+		if err != nil {
+			continue
+		}
+		for _, rr := range r.Answer {
+			if c, ok := rr.(*dns.CNAME); ok {
+				return strings.ToLower(c.Target)
+			}
+		}
+		return ""
+	}
+	return ""
+}
+
+// lookupNS asks the resolvers for a zone's name servers.
+func (m *Manager) lookupNS(ctx context.Context, zone string) ([]string, []string, error) {
+	var queries []string
+	var last error = errors.New("no resolvers")
+	for _, res := range m.cfg.ACME.Resolvers {
+		r, q, err := exchange(ctx, res, zone, dns.TypeNS, true)
+		queries = append(queries, q)
+		if err != nil {
+			last = err
+			continue
+		}
+		var out []string
+		for _, rr := range r.Answer {
+			if ns, ok := rr.(*dns.NS); ok {
+				out = append(out, strings.ToLower(ns.Ns))
+			}
+		}
+		if len(out) > 0 {
+			sort.Strings(out)
+			return out, queries, nil
+		}
+		last = fmt.Errorf("%s answered %s", res, dns.RcodeToString[r.Rcode])
+	}
+	return nil, queries, last
+}
+
+// lookupAddrs asks the resolvers for a host's addresses.
+func (m *Manager) lookupAddrs(ctx context.Context, host string) ([]string, []string) {
+	var queries []string
+	for _, res := range m.cfg.ACME.Resolvers {
+		var out []string
+		for _, t := range []uint16{dns.TypeA, dns.TypeAAAA} {
+			r, q, err := exchange(ctx, res, host, t, true)
+			queries = append(queries, q)
+			if err != nil {
+				continue
+			}
+			for _, rr := range r.Answer {
+				switch a := rr.(type) {
+				case *dns.A:
+					out = append(out, a.A.String())
+				case *dns.AAAA:
+					out = append(out, a.AAAA.String())
+				}
+			}
+		}
+		if len(out) > 0 {
+			return out, queries
+		}
+	}
+	return nil, queries
+}

internal/certs/check_test.go

@@ -0,0 +1,266 @@
+package certs
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"steward/internal/config"
+	"steward/internal/jarl"
+)
+
+// fakeJarl is Jarl's API as far as Steward uses it, and the DNS server
+// that serves what it writes (both the "public resolver" and the zone's
+// name server).
+type fakeJarl struct {
+	mu        sync.Mutex
+	txt       map[string][]string // name -> values
+	canWrite  bool
+	token     string
+	served    bool // the DNS server serves what is written
+	zones     []jarl.Zone
+	writes    int
+	dryRuns   int
+	nsHasIPv6 bool
+}
+
+func (f *fakeJarl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Header.Get("Authorization") != "Bearer "+f.token {
+		w.WriteHeader(401)
+		w.Write([]byte(`{"error":{"code":"unauthenticated","message":"invalid API token: unknown token"}}`))
+		return
+	}
+	switch r.URL.Path {
+	case "/api/v1/auth/whoami":
+		w.Write([]byte(`{"user":{"username":"steward"},"build":{"label":"v0.6.1"}}`))
+	case "/api/v1/zones":
+		json.NewEncoder(w).Encode(f.zones)
+	case "/api/v1/apply":
+		var b struct {
+			Ops []struct {
+				Op   string         `json:"op"`
+				Data map[string]any `json:"data"`
+			} `json:"ops"`
+			DryRun bool `json:"dry_run"`
+		}
+		json.NewDecoder(r.Body).Decode(&b)
+		if !f.canWrite {
+			w.WriteHeader(403)
+			w.Write([]byte(`{"error":{"code":"forbidden","message":"permission denied: this needs records:write"}}`))
+			return
+		}
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if b.DryRun {
+			f.dryRuns++
+			w.Write([]byte(`{}`))
+			return
+		}
+		f.writes++
+		for _, op := range b.Ops {
+			name, v := op.Data["name"].(string), op.Data["data"].(string)
+			var keep []string
+			for _, x := range f.txt[name] {
+				if x != v {
+					keep = append(keep, x)
+				}
+			}
+			if op.Op == "record.create" {
+				keep = append(keep, v)
+			}
+			f.txt[name] = keep
+		}
+		w.Write([]byte(`{}`))
+	default:
+		w.WriteHeader(404)
+	}
+}
+
+func (f *fakeJarl) dns(w dns.ResponseWriter, r *dns.Msg) {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	q := r.Question[0]
+	hdr := dns.RR_Header{Name: q.Name, Class: dns.ClassINET, Ttl: 60}
+	switch q.Qtype {
+	case dns.TypeNS:
+		if q.Name == "adbraun.test." {
+			hdr.Rrtype = dns.TypeNS
+			m.Answer = append(m.Answer, &dns.NS{Hdr: hdr, Ns: "ns1.adbraun.test."})
+		} else {
+			m.Rcode = dns.RcodeNameError
+		}
+	case dns.TypeA:
+		hdr.Rrtype = dns.TypeA
+		m.Answer = append(m.Answer, &dns.A{Hdr: hdr, A: net.ParseIP("127.0.0.1")})
+	case dns.TypeAAAA:
+		// An IPv6 address nothing answers on (the check must not fail
+		// because of it).
+		if f.nsHasIPv6 {
+			hdr.Rrtype = dns.TypeAAAA
+			m.Answer = append(m.Answer, &dns.AAAA{Hdr: hdr, AAAA: net.ParseIP("100::1")})
+		}
+	case dns.TypeCNAME:
+		if q.Name == "_acme-challenge.dhcp.other.test." {
+			hdr.Rrtype = dns.TypeCNAME
+			m.Answer = append(m.Answer, &dns.CNAME{Hdr: hdr, Target: "dhcp.acme.adbraun.test."})
+		}
+	case dns.TypeTXT:
+		f.mu.Lock()
+		if f.served {
+			for _, v := range f.txt[q.Name] {
+				hdr.Rrtype = dns.TypeTXT
+				m.Answer = append(m.Answer, &dns.TXT{Hdr: hdr, Txt: []string{v}})
+			}
+		}
+		f.mu.Unlock()
+	}
+	w.WriteMsg(m)
+}
+
+func setup(t *testing.T, f *fakeJarl, edit func(a *config.ACME)) (*Manager, string) {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(f.dns)}
+	go srv.ActivateAndServe()
+	t.Cleanup(func() { srv.Shutdown() })
+	hs := httptest.NewServer(f)
+	t.Cleanup(hs.Close)
+	addr := pc.LocalAddr().String()
+	_, port, _ := net.SplitHostPort(addr)
+	oldPort, oldWait, oldPoll := nsPort, propagationWait, pollEvery
+	nsPort, propagationWait, pollEvery = port, 2*time.Second, 100*time.Millisecond
+	t.Cleanup(func() { nsPort, propagationWait, pollEvery = oldPort, oldWait, oldPoll })
+	a := config.ACME{Domain: "dhcp.adbraun.test", Resolvers: []string{addr}, CheckPropagation: true}
+	if edit != nil {
+		edit(&a)
+	}
+	jc, _ := jarl.New(hs.URL, "steward_good", "", false)
+	m := &Manager{cfg: Config{ACME: a}, jarl: jc, now: time.Now}
+	return m, addr
+}
+
+func steps(c *Check) string {
+	var out []string
+	for _, s := range c.Steps {
+		out = append(out, s.Name+"="+s.Status)
+	}
+	return strings.Join(out, " ")
+}
+
+func newFake() *fakeJarl {
+	return &fakeJarl{txt: map[string][]string{}, canWrite: true, token: "steward_good", served: true,
+		zones: []jarl.Zone{{Name: "adbraun.test.", View: "internal", LinkGroup: 7}, {Name: "adbraun.test.", View: "external", LinkGroup: 7}, {Name: "other.test.", View: "internal"}}}
+}
+
+func TestPreflightOK(t *testing.T) {
+	f := newFake()
+	f.nsHasIPv6 = true
+	m, _ := setup(t, f, nil)
+	c := m.Preflight(context.Background())
+	want := "Jarl=ok Zone=ok Permission=ok Write=ok Name servers=ok Public resolvers=ok Clean up=ok"
+	if !c.OK || steps(c) != want {
+		t.Fatalf("got %s\n%+v", steps(c), c.Steps)
+	}
+	if !strings.Contains(c.Steps[1].Detail, "views internal, external") {
+		t.Errorf("zone step: %s", c.Steps[1].Detail)
+	}
+	// Linked zones are written once; the test record is gone afterwards.
+	if f.dryRuns != 1 || f.writes != 2 || len(f.txt["_acme-challenge.dhcp.adbraun.test."]) != 0 {
+		t.Errorf("dry runs %d writes %d records %v", f.dryRuns, f.writes, f.txt)
+	}
+}
+
+func TestPreflightFailures(t *testing.T) {
+	cases := []struct {
+		name string
+		edit func(f *fakeJarl, a *config.ACME)
+		want string
+		text string
+	}{
+		{"bad token", func(f *fakeJarl, a *config.ACME) { f.token = "other" }, "Jarl=fail", "check jarl.token"},
+		{"no zone", func(f *fakeJarl, a *config.ACME) { a.Domain = "dhcp.example.net" }, "Jarl=ok Zone=fail", "no primary zone the token can see"},
+		{"no permission", func(f *fakeJarl, a *config.ACME) { f.canWrite = false }, "Jarl=ok Zone=ok Permission=fail",
+			"records:write zones=adbraun.test names=_acme-challenge.dhcp.adbraun.test types=TXT"},
+		{"not served", func(f *fakeJarl, a *config.ACME) { f.served = false },
+			"Jarl=ok Zone=ok Permission=ok Write=ok Name servers=fail Public resolvers=fail Clean up=ok", "still don't serve it"},
+		{"not delegated", func(f *fakeJarl, a *config.ACME) { f.zones[0].Name, f.zones[1].Name = "test.", "test." },
+			"Jarl=ok Zone=ok Permission=ok Write=ok Name servers=fail Public resolvers=ok Clean up=ok", "know no name servers for test"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFake()
+			var am *config.ACME
+			m, _ := setup(t, f, func(a *config.ACME) { am = a })
+			tc.edit(f, am)
+			m.cfg.ACME = *am
+			c := m.Preflight(context.Background())
+			if c.OK || steps(c) != tc.want {
+				t.Fatalf("got %s, want %s\n%+v", steps(c), tc.want, c.Steps)
+			}
+			all := ""
+			for _, s := range c.Steps {
+				all += s.Detail + "\n"
+			}
+			if !strings.Contains(all, tc.text) {
+				t.Errorf("details lack %q:\n%s", tc.text, all)
+			}
+		})
+	}
+}
+
+func TestResolversOnlyWarnWhenNameServersServe(t *testing.T) {
+	f := newFake()
+	m, _ := setup(t, f, nil)
+	// The "public resolver" is another server that never sees the record.
+	pc, _ := net.ListenPacket("udp", "127.0.0.1:0")
+	srv := &dns.Server{PacketConn: pc, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
+		mm := new(dns.Msg)
+		mm.SetReply(r)
+		if r.Question[0].Qtype == dns.TypeTXT {
+			mm.Rcode = dns.RcodeNameError
+		} else {
+			f.dns(w, r)
+			return
+		}
+		w.WriteMsg(mm)
+	})}
+	go srv.ActivateAndServe()
+	defer srv.Shutdown()
+	m.cfg.ACME.Resolvers = []string{pc.LocalAddr().String()}
+	c := m.Preflight(context.Background())
+	if !c.OK || !strings.Contains(steps(c), "Public resolvers=warn") {
+		t.Fatalf("got %s %+v", steps(c), c.Steps)
+	}
+}
+
+func TestPreflightFollowsCNAME(t *testing.T) {
+	f := newFake()
+	m, _ := setup(t, f, func(a *config.ACME) { a.Domain = "dhcp.other.test" })
+	c := m.Preflight(context.Background())
+	if !c.OK || !strings.Contains(c.Steps[2].Detail, "dhcp.acme.adbraun.test") {
+		t.Fatalf("got %s %+v", steps(c), c.Steps)
+	}
+}
+
+func TestZonesFor(t *testing.T) {
+	zs := []jarl.Zone{{Name: "lol.", View: "a"}, {Name: "adbraun.lol.", View: "internal", LinkGroup: 3}, {Name: "adbraun.lol.", View: "external", LinkGroup: 3},
+		{Name: "adbraun.lol.", View: "guest"}, {Name: "notadbraun.lol.", View: "internal"}}
+	got := jarl.ZonesFor("_acme-challenge.dhcp.adbraun.lol.", zs, "")
+	if len(got) != 2 || got[0].View != "internal" || got[1].View != "guest" {
+		t.Errorf("got %+v", got)
+	}
+	if n := jarl.ChallengeName("*.Dhcp.adbraun.lol."); n != "_acme-challenge.dhcp.adbraun.lol." {
+		t.Errorf("challenge name %q", n)
+	}
+}

internal/certs/manager.go

@@ -0,0 +1,484 @@
+// Package certs obtains and renews the web UI's certificate from an ACME CA
+// (Let's Encrypt), with Jarl publishing the DNS-01 challenge
+// (docs/certificates.md). Before every order a preflight check proves the
+// challenge can be published and seen, so a broken setup never spends the
+// CA's rate limits.
+package certs
+
+import (
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-acme/lego/v4/certificate"
+	"github.com/go-acme/lego/v4/challenge/dns01"
+	"github.com/go-acme/lego/v4/lego"
+	legolog "github.com/go-acme/lego/v4/log"
+	"github.com/go-acme/lego/v4/registration"
+
+	"steward/internal/api"
+	"steward/internal/config"
+	"steward/internal/jarl"
+)
+
+// Store keeps the account and certificate (the database's meta table).
+type Store interface {
+	GetMeta(ctx context.Context, key string) (string, error)
+	SetMeta(ctx context.Context, key, value string) error
+}
+
+const (
+	metaAccount = "acme.account"
+	metaCert    = "acme.certificate"
+)
+
+// Config configures the manager.
+type Config struct {
+	ACME    config.ACME
+	Version string
+}
+
+// Status is api.CertStatus.
+type Status = api.CertStatus
+
+// Manager keeps the web UI's certificate.
+type Manager struct {
+	cfg   Config
+	jarl  *jarl.Client
+	store Store
+	log   *slog.Logger
+	now   func() time.Time
+
+	mu          sync.Mutex
+	cert        *tls.Certificate
+	leaf        *x509.Certificate
+	issuedBy    string
+	working     bool
+	lastErr     string
+	lastAttempt time.Time
+	nextAttempt time.Time
+	lastCheck   *Check
+	failures    int
+	force       bool // renew now, even if not due (Kick)
+
+	lego sync.Mutex // one ACME order at a time
+	kick chan struct{}
+}
+
+// New makes a manager and loads the stored certificate.
+func New(ctx context.Context, cfg Config, j *jarl.Client, st Store, log *slog.Logger) *Manager {
+	if log == nil {
+		log = slog.Default()
+	}
+	// lego logs each step of an order; send it to the journal with the rest.
+	legolog.Logger = slog.NewLogLogger(log.With("component", "acme").Handler(), slog.LevelInfo)
+	m := &Manager{cfg: cfg, jarl: j, store: st, log: log, now: time.Now, kick: make(chan struct{}, 1)}
+	m.load(ctx)
+	return m
+}
+
+type stored struct {
+	Domain    string    `json:"domain"`
+	Directory string    `json:"directory"`
+	CertPEM   string    `json:"cert_pem"`
+	KeyPEM    string    `json:"key_pem"`
+	Obtained  time.Time `json:"obtained"`
+}
+
+func (m *Manager) load(ctx context.Context) {
+	raw, err := m.store.GetMeta(ctx, metaCert)
+	if err != nil || raw == "" {
+		return
+	}
+	var s stored
+	if json.Unmarshal([]byte(raw), &s) != nil {
+		return
+	}
+	if err := m.use(s); err != nil {
+		m.log.Warn("the stored certificate can't be used; getting a new one", "err", err)
+	}
+}
+
+// use makes a stored certificate the one served.
+func (m *Manager) use(s stored) error {
+	c, err := tls.X509KeyPair([]byte(s.CertPEM), []byte(s.KeyPEM))
+	if err != nil {
+		return err
+	}
+	leaf, err := x509.ParseCertificate(c.Certificate[0])
+	if err != nil {
+		return err
+	}
+	c.Leaf = leaf
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	// A certificate for another name isn't served.
+	if !strings.EqualFold(s.Domain, m.cfg.ACME.Domain) {
+		m.cert, m.leaf = nil, nil
+		return fmt.Errorf("it is for %s, not %s", s.Domain, m.cfg.ACME.Domain)
+	}
+	m.cert, m.leaf, m.issuedBy = &c, leaf, s.Directory
+	return nil
+}
+
+// Certificate is the certificate to serve (nil when there is none yet).
+func (m *Manager) Certificate() *tls.Certificate {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.cert == nil || m.now().After(m.leaf.NotAfter) {
+		return nil
+	}
+	return m.cert
+}
+
+// Status describes the certificate.
+func (m *Manager) Status() Status {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	st := Status{Domain: m.cfg.ACME.Domain, Directory: m.cfg.ACME.DirectoryURL(), Staging: m.cfg.ACME.Staging(), State: "none",
+		IssuedBy: m.issuedBy, LastError: m.lastErr, Check: m.lastCheck}
+	if m.leaf != nil {
+		nb, na := m.leaf.NotBefore, m.leaf.NotAfter
+		ra := m.renewAt()
+		st.NotBefore, st.NotAfter, st.RenewAt, st.Issuer, st.State = &nb, &na, &ra, m.leaf.Issuer.CommonName, "ok"
+	}
+	if m.lastErr != "" {
+		st.State = "error"
+	}
+	if m.working {
+		st.State = "working"
+	}
+	if !m.lastAttempt.IsZero() {
+		t := m.lastAttempt
+		st.LastAttempt = &t
+	}
+	if !m.nextAttempt.IsZero() {
+		t := m.nextAttempt
+		st.NextAttempt = &t
+	}
+	return st
+}
+
+// renewAt is when the certificate is renewed: a third of its lifetime
+// before it ends (30 days for Let's Encrypt's 90), m.mu held.
+func (m *Manager) renewAt() time.Time {
+	life := m.leaf.NotAfter.Sub(m.leaf.NotBefore)
+	return m.leaf.NotAfter.Add(-life / 3).UTC().Truncate(time.Second)
+}
+
+// due reports whether a new certificate is needed: none yet, another
+// directory configured, or renewal time.
+func (m *Manager) due() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.force || m.leaf == nil || m.issuedBy != m.cfg.ACME.DirectoryURL() || !m.now().Before(m.renewAt())
+}
+
+// Run keeps the certificate: it gets one when needed, checks twice a day
+// whether it is due, and retries failures with a growing wait.
+func (m *Manager) Run(ctx context.Context) {
+	timer := time.NewTimer(5 * time.Second)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-m.kick:
+		case <-timer.C:
+		}
+		wait := 12 * time.Hour
+		if m.due() {
+			if err := m.Renew(ctx); err != nil {
+				m.mu.Lock()
+				m.failures++
+				// 1h, 3h, 6h, then 12h.
+				waits := []time.Duration{time.Hour, 3 * time.Hour, 6 * time.Hour, 12 * time.Hour}
+				wait = waits[min(m.failures, len(waits))-1]
+				m.mu.Unlock()
+				m.log.Warn("getting the web UI's certificate failed; trying again later", "domain", m.cfg.ACME.Domain, "err", err, "retry_in", wait)
+			}
+		}
+		m.mu.Lock()
+		m.nextAttempt = m.now().Add(wait).UTC()
+		m.mu.Unlock()
+		if !timer.Stop() {
+			select {
+			case <-timer.C:
+			default:
+			}
+		}
+		timer.Reset(wait)
+	}
+}
+
+// Kick asks Run to renew now, even if the certificate isn't due.
+func (m *Manager) Kick() {
+	m.mu.Lock()
+	m.force, m.failures = true, 0
+	m.mu.Unlock()
+	select {
+	case m.kick <- struct{}{}:
+	default:
+	}
+}
+
+// ErrBusy is returned while another order runs.
+var ErrBusy = errors.New("a certificate order is already running")
+
+// Renew gets a new certificate now: the preflight check, then the order.
+func (m *Manager) Renew(ctx context.Context) error {
+	if !m.lego.TryLock() {
+		return ErrBusy
+	}
+	defer m.lego.Unlock()
+	m.mu.Lock()
+	m.working, m.lastAttempt = true, m.now().UTC()
+	m.mu.Unlock()
+	err := m.renew(ctx)
+	m.mu.Lock()
+	m.working, m.force = false, false
+	if err != nil {
+		m.lastErr = err.Error()
+	} else {
+		m.lastErr, m.failures = "", 0
+	}
+	m.mu.Unlock()
+	return err
+}
+
+func (m *Manager) renew(ctx context.Context) error {
+	c := m.Preflight(ctx)
+	if !c.OK {
+		for _, s := range c.Steps {
+			if s.Status == "fail" {
+				return fmt.Errorf("the check before asking the CA failed at %s: %s", strings.ToLower(s.Name), s.Detail)
+			}
+		}
+	}
+	m.log.Info("the check passed; asking the CA for a certificate", "domain", m.cfg.ACME.Domain, "directory", m.cfg.ACME.DirectoryURL())
+	res, err := m.obtain(ctx)
+	if err != nil {
+		return err
+	}
+	s := stored{Domain: m.cfg.ACME.Domain, Directory: m.cfg.ACME.DirectoryURL(), CertPEM: string(res.Certificate), KeyPEM: string(res.PrivateKey), Obtained: m.now().UTC()}
+	if err := m.use(s); err != nil {
+		return fmt.Errorf("the CA's certificate can't be used: %w", err)
+	}
+	b, _ := json.Marshal(s)
+	if err := m.store.SetMeta(ctx, metaCert, string(b)); err != nil {
+		return err
+	}
+	m.mu.Lock()
+	na := m.leaf.NotAfter
+	m.mu.Unlock()
+	m.log.Info("got a new certificate for the web UI", "domain", s.Domain, "until", na.Format(time.RFC3339), "staging", m.cfg.ACME.Staging())
+	return nil
+}
+
+// ---- ACME
+
+type account struct {
+	Directory string `json:"directory"`
+	Email     string `json:"email,omitempty"`
+	URI       string `json:"uri,omitempty"`
+	KeyPEM    string `json:"key_pem"`
+}
+
+type legoUser struct {
+	email string
+	reg   *registration.Resource
+	key   crypto.PrivateKey
+}
+
+func (u *legoUser) GetEmail() string                        { return u.email }
+func (u *legoUser) GetRegistration() *registration.Resource { return u.reg }
+func (u *legoUser) GetPrivateKey() crypto.PrivateKey        { return u.key }
+
+func (m *Manager) httpClient() (*http.Client, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
+	if f := m.cfg.ACME.CABundle; f != "" {
+		pemData, err := os.ReadFile(f)
+		if err != nil {
+			return nil, fmt.Errorf("tls.acme.ca_bundle: %w", err)
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil || pool == nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(pemData) {
+			return nil, fmt.Errorf("tls.acme.ca_bundle: no certificates in %s", f)
+		}
+		tr.TLSClientConfig.RootCAs = pool
+	}
+	return &http.Client{Timeout: 2 * time.Minute, Transport: tr}, nil
+}
+
+// client returns a lego client for the account, registering it if needed.
+func (m *Manager) client(ctx context.Context) (*lego.Client, error) {
+	dir := m.cfg.ACME.DirectoryURL()
+	hc, err := m.httpClient()
+	if err != nil {
+		return nil, err
+	}
+	user := &legoUser{email: m.cfg.ACME.Email}
+	var a account
+	if raw, _ := m.store.GetMeta(ctx, metaAccount); raw != "" && json.Unmarshal([]byte(raw), &a) == nil && a.Directory == dir && a.KeyPEM != "" {
+		b, _ := pem.Decode([]byte(a.KeyPEM))
+		if b == nil {
+			return nil, errors.New("the stored ACME account key isn't PEM")
+		}
+		if user.key, err = x509.ParsePKCS8PrivateKey(b.Bytes); err != nil {
+			return nil, fmt.Errorf("the stored ACME account key: %w", err)
+		}
+		if a.URI != "" {
+			user.reg = &registration.Resource{URI: a.URI}
+		}
+	} else {
+		if user.key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader); err != nil {
+			return nil, err
+		}
+	}
+	cfg := lego.NewConfig(user)
+	cfg.CADirURL, cfg.HTTPClient, cfg.UserAgent = dir, hc, "steward/"+m.cfg.Version
+	cfg.Certificate.Timeout = 90 * time.Second
+	c, err := lego.NewClient(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("the ACME server %s: %w", dir, err)
+	}
+	if user.reg != nil {
+		return c, nil
+	}
+	if !m.cfg.ACME.AgreeTOS {
+		tos := c.GetToSURL()
+		if tos == "" {
+			tos = "on the CA's website"
+		}
+		return nil, fmt.Errorf("Steward registers an account with %s only after you accept its terms of service (%s): set tls.acme.agree_tos: true in steward.yaml and restart stewardd", dir, tos)
+	}
+	reg, err := c.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: true})
+	if err != nil {
+		return nil, fmt.Errorf("registering the ACME account: %w", err)
+	}
+	user.reg = reg
+	der, err := x509.MarshalPKCS8PrivateKey(user.key)
+	if err != nil {
+		return nil, err
+	}
+	b, _ := json.Marshal(account{Directory: dir, Email: user.email, URI: reg.URI,
+		KeyPEM: string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))})
+	if err := m.store.SetMeta(ctx, metaAccount, string(b)); err != nil {
+		return nil, err
+	}
+	m.log.Info("ACME account registered", "directory", dir, "account", reg.URI)
+	return c, nil
+}
+
+// provider publishes the challenge through Jarl.
+type provider struct {
+	m   *Manager
+	ctx context.Context
+}
+
+func (p *provider) set(domain, keyAuth string, set bool) error {
+	info := dns01.GetChallengeInfo(domain, keyAuth)
+	ctx := p.ctx
+	if !set {
+		ctx = context.WithoutCancel(ctx)
+	}
+	all, err := p.m.jarl.Zones(ctx)
+	if err != nil {
+		return err
+	}
+	zones := jarl.ZonesFor(info.EffectiveFQDN, all, "")
+	if len(zones) == 0 {
+		return fmt.Errorf("no primary zone in Jarl holds %s", strings.TrimSuffix(info.EffectiveFQDN, "."))
+	}
+	return p.m.jarl.SetTXT(ctx, info.EffectiveFQDN, info.Value, zones, set, false, "Steward's certificate")
+}
+
+func (p *provider) Present(domain, token, keyAuth string) error { return p.set(domain, keyAuth, true) }
+func (p *provider) CleanUp(domain, token, keyAuth string) error { return p.set(domain, keyAuth, false) }
+
+// Timeout is how long lego waits for the record to reach the name servers.
+func (p *provider) Timeout() (timeout, interval time.Duration) {
+	return 5 * time.Minute, 5 * time.Second
+}
+
+func (m *Manager) obtain(ctx context.Context) (*certificate.Resource, error) {
+	c, err := m.client(ctx)
+	if err != nil {
+		return nil, err
+	}
+	opts := []dns01.ChallengeOption{dns01.AddDNSTimeout(10 * time.Second)}
+	if len(m.cfg.ACME.Resolvers) > 0 {
+		opts = append(opts, dns01.AddRecursiveNameservers(m.cfg.ACME.Resolvers))
+	}
+	if !m.cfg.ACME.CheckPropagation {
+		opts = append(opts, dns01.DisableAuthoritativeNssPropagationRequirement())
+	}
+	if err := c.Challenge.SetDNS01Provider(&provider{m: m, ctx: ctx}, opts...); err != nil {
+		return nil, err
+	}
+	res, err := c.Certificate.Obtain(certificate.ObtainRequest{Domains: []string{m.cfg.ACME.Domain}, Bundle: true})
+	if err != nil {
+		return nil, cleanLegoError(err)
+	}
+	return res, nil
+}
+
+// cleanLegoError shortens lego's errors (as Jarl does): "[name] acme:
+// error: 403 :: urn:ietf:params:acme:error:unauthorized :: MSG" becomes
+// "name: the CA refused (unauthorized): MSG".
+func cleanLegoError(err error) error {
+	msg := strings.TrimSpace(strings.Replace(err.Error(), "error: one or more domains had a problem:", "", 1))
+	var out []string
+	for _, line := range strings.Split(msg, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		domain := ""
+		for strings.HasPrefix(line, "[") {
+			end := strings.Index(line, "] ")
+			if end < 0 {
+				break
+			}
+			if domain == "" {
+				domain = line[1:end]
+			}
+			line = line[end+2:]
+		}
+		for _, p := range []string{"acme: error presenting token: ", "acme: error: ", "acme: "} {
+			line = strings.TrimPrefix(line, p)
+		}
+		if mm := problemRe.FindStringSubmatch(line); mm != nil {
+			line = "the CA refused (" + mm[2] + "): " + mm[3]
+		}
+		if domain != "" {
+			line = domain + ": " + line
+		}
+		out = append(out, line)
+	}
+	if len(out) == 0 {
+		return err
+	}
+	return errors.New(strings.Join(out, "; "))
+}
+
+var problemRe = regexp.MustCompile(`^(\d+) :: urn:ietf:params:acme:error:(\w+) :: (.*)$`)