Add backups, token expiry warnings and name server checks

- Daily database backups: jarl-backup.timer runs 'jarld backup', which
  writes jarl-<UTC time>.db into backup.dir, checks it before naming it,
  and prunes by the time in the name while always keeping the newest
  keep_at_least. New backup: section in jarl.yaml; the dashboard and
  'jarl status' show the newest backup and warn when it is stale or
  failed. The Debian package enables the timer.
- 'jarld restore [FILE | latest]': checks the backup, shows it next to
  the current database, keeps the current one aside and records the
  restore. 'jarld serve' holds a database lock, so a restore can't run
  under a live jarld and two jarlds can't share a database.
- Token expiry warnings from 14 days ahead: dashboard, token lists, a
  daily log line, and a CLI warning via the Jarl-Token-Expires header;
  'jarl token list --expiring'.
- 'jarl zone serials' and the zone page's Name servers tab: compare the
  serial on a primary zone's secondaries (NS records, also-notify) or a
  secondary zone's primaries with this server's.
- 'jarl zone delegation': for zones on the internet, check the parent's
  delegation and glue against the zone; an undelegated zone gets a note.
- Server: Jarl/<version> header; CLI exit code 2 for failed checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/jarl/admin_cmds.go

@@ -17,6 +17,7 @@ import (
 
 	"jarl/internal/api"
 	"jarl/internal/client"
+	"jarl/internal/model"
 )
 
 // randomSecret returns a base64 TSIG secret as long as the algorithm's
@@ -71,6 +72,37 @@ func readNewPassword() (string, error) {
 	return pw, nil
 }
 
+// fmtExpiry shows a token's expiry, flagging one that is close or past.
+func fmtExpiry(exp, revoked *time.Time) string {
+	s := fmtTime(exp)
+	if exp == nil || revoked != nil {
+		return s
+	}
+	switch d := time.Until(*exp); {
+	case d < 0:
+		return s + " (EXPIRED)"
+	case d < model.TokenExpiryWarning:
+		return s + " (in " + fmtUntil(d) + ")"
+	}
+	return s
+}
+
+// parseDays parses a duration such as 14d, 36h or 90m.
+func parseDays(s string) (time.Duration, error) {
+	if n, ok := strings.CutSuffix(s, "d"); ok {
+		days, err := strconv.Atoi(n)
+		if err != nil || days <= 0 {
+			return 0, fmt.Errorf("invalid duration %q (use e.g. 14d or 36h)", s)
+		}
+		return time.Duration(days) * 24 * time.Hour, nil
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil || d <= 0 {
+		return 0, fmt.Errorf("invalid duration %q (use e.g. 14d or 36h)", s)
+	}
+	return d, nil
+}
+
 func fmtTime(t *time.Time) string {
 	if t == nil || t.IsZero() {
 		return "-"
@@ -152,6 +184,22 @@ func statusCmd() *cobra.Command {
 				for _, d := range st.Drift {
 					fmt.Fprintf(w, "Drift:\t%s %s (fix with 'jarl resync --force' or apply with --force)\n", d.Path, d.Reason)
 				}
+				if b := resp.Backup; b != nil {
+					switch {
+					case b.LastAt == nil:
+						fmt.Fprintf(w, "Backup:\tnone recorded (see 'jarld backup --help')\n")
+					case b.Stale:
+						fmt.Fprintf(w, "Backup:\tLAST ONE %s AGO: %s (is jarl-backup.timer running?)\n", strings.ToUpper(fmtUntil(time.Since(*b.LastAt))), b.LastFile)
+					default:
+						fmt.Fprintf(w, "Backup:\t%s ago: %s\n", fmtUntil(time.Since(*b.LastAt)), b.LastFile)
+					}
+					if b.LastError != "" {
+						fmt.Fprintf(w, "Backup:\tFAILED %s: %s\n", fmtTime(b.LastErrorAt), b.LastError)
+					}
+				}
+				for _, t := range resp.ExpiringTokens {
+					fmt.Fprintf(w, "Token:\t%s/%s (id %d) expires %s\n", t.Username, t.Name, t.ID, fmtExpiry(t.ExpiresAt, nil))
+				}
 				if len(st.Problems) > 0 {
 					w.(interface{ Flush() error }).Flush()
 					fmt.Println("Problems:")
@@ -506,12 +554,23 @@ func userCmd() *cobra.Command {
 
 func tokenCmd() *cobra.Command {
 	cmd := &cobra.Command{Use: "token", Aliases: []string{"tokens"}, Short: "Manage API tokens", Long: "Manage API tokens. A token has its user's permissions, or fewer when it has\nits own grants.\n\n" + grantHelp}
-	var user string
+	var user, expiring string
 	var all, revoked bool
 	list := &cobra.Command{
 		Use:   "list",
 		Short: "List your tokens (--user or --all for others; admin)",
+		Example: `  jarl token list
+  jarl token list --all --expiring        # every token that expires within 14 days
+  jarl token list --all --expiring 60d`,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			var within time.Duration
+			if expiring != "" {
+				d, err := parseDays(expiring)
+				if err != nil {
+					return fmt.Errorf("--expiring: %w", err)
+				}
+				within = d
+			}
 			c, err := newClient()
 			if err != nil {
 				return err
@@ -535,6 +594,17 @@ func tokenCmd() *cobra.Command {
 			if err := c.Get(ctx(), path, qv, &tokens); err != nil {
 				return err
 			}
+			if within > 0 {
+				// Expiring soon, or already expired (and not revoked).
+				now := time.Now()
+				kept := []api.Token{}
+				for _, t := range tokens {
+					if t.ExpiresWithin(now, within) {
+						kept = append(kept, t)
+					}
+				}
+				tokens = kept
+			}
 			return output(tokens, func(w io.Writer) {
 				fmt.Fprintln(w, "ID\tUSER\tNAME\tPREFIX\tGRANTS\tEXPIRES\tROTATED\tLAST USED\tREVOKED")
 				for _, t := range tokens {
@@ -546,7 +616,7 @@ func tokenCmd() *cobra.Command {
 					if t.PrevExpiresAt != nil && time.Now().Before(*t.PrevExpiresAt) {
 						rot += " (old secret until " + fmtTime(t.PrevExpiresAt) + ")"
 					}
-					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", t.ID, t.Username, t.Name, t.PublicID, g, fmtTime(t.ExpiresAt), rot, fmtTime(t.LastUsedAt), fmtTime(t.RevokedAt))
+					fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", t.ID, t.Username, t.Name, t.PublicID, g, fmtExpiry(t.ExpiresAt, t.RevokedAt), rot, fmtTime(t.LastUsedAt), fmtTime(t.RevokedAt))
 				}
 			})
 		},
@@ -554,6 +624,8 @@ func tokenCmd() *cobra.Command {
 	list.Flags().StringVar(&user, "user", "", "tokens of this user")
 	list.Flags().BoolVar(&all, "all", false, "tokens of every user")
 	list.Flags().BoolVar(&revoked, "revoked", false, "include revoked tokens")
+	list.Flags().StringVar(&expiring, "expiring", "", "only tokens that expire within this long, or have expired (default 14d)")
+	list.Flags().Lookup("expiring").NoOptDefVal = "14d"
 	var role, expires string
 	var grants []string
 	var save bool

cmd/jarl/main.go

@@ -13,7 +13,9 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"text/tabwriter"
+	"time"
 
 	"github.com/spf13/cobra"
 	"go.yaml.in/yaml/v3"
@@ -154,9 +156,45 @@ func newClient() (*client.Client, error) {
 		}
 	}
 	c.DryRun, c.Force = flagDryRun || c.Preflight, flagForce
+	if !c.Preflight {
+		c.OnTokenExpiry = tokenExpiryWarning(t.Name)
+	}
 	return c, nil
 }
 
+// tokenExpiryWarning returns a function that warns, once, that the token
+// used for server expires soon. Scripts (certbot hooks, cron jobs) show it
+// in their logs well before the token stops working.
+func tokenExpiryWarning(server string) func(time.Time) {
+	var once sync.Once
+	return func(t time.Time) {
+		once.Do(func() {
+			where := ""
+			if server != "" && server != localName {
+				where = " for server " + server
+			}
+			when := "expires " + fmtTime(&t) + " (in " + fmtUntil(time.Until(t)) + ")"
+			if time.Now().After(t) {
+				when = "has expired"
+			}
+			fmt.Fprintf(os.Stderr, "warning: the API token%s %s; give it a new lifetime with 'jarl token rotate --current' (see 'jarl token rotate --help')\n", where, when)
+		})
+	}
+}
+
+// fmtUntil is a short, rounded-down duration: "5 days", "3 hours".
+func fmtUntil(d time.Duration) string {
+	switch {
+	case d >= 48*time.Hour:
+		return fmt.Sprintf("%d days", int(d/(24*time.Hour)))
+	case d >= 2*time.Hour:
+		return fmt.Sprintf("%d hours", int(d/time.Hour))
+	case d >= time.Minute:
+		return fmt.Sprintf("%d minutes", int(d/time.Minute))
+	}
+	return "under a minute"
+}
+
 func firstNonEmpty(vals ...string) string {
 	for _, v := range vals {
 		if v != "" {
@@ -326,6 +364,10 @@ func printError(err error) {
 // authenticated or not allowed, 4 not found, 5 conflict, 6 invalid
 // request or rejected by validation, 7 server error.
 func exitCode(err error) int {
+	var cf *checkFailed
+	if errors.As(err, &cf) {
+		return 2
+	}
 	var ce *client.Error
 	if errors.As(err, &ce) {
 		switch {

cmd/jarl/nscheck.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"jarl/internal/api"
+	"jarl/internal/nscheck"
+)
+
+// checkFailed is returned when a check ran but found problems (exit code
+// 2), after its report was printed.
+type checkFailed struct{ msg string }
+
+func (e *checkFailed) Error() string { return e.msg }
+
+func zoneSerialsCmd() *cobra.Command {
+	var view string
+	cmd := &cobra.Command{
+		Use:   "serials ZONE",
+		Short: "Compare the zone's serial on its other name servers with this server's",
+		Long: `Asks the zone's other name servers which serial they serve and compares it
+with the one this server serves.
+
+For a primary zone those are the servers in its NS records and its
+also-notify targets (its secondaries); for a secondary or mirror zone, its
+primaries. Addresses come from Jarl's own A/AAAA records where it has them,
+otherwise from a DNS lookup. This server itself isn't asked.
+
+Exits with 2 when a server is behind, doesn't serve the zone, refuses, or
+doesn't answer, so it can run from cron or a monitoring system.`,
+		Example: `  jarl zone serials adbraun.lol
+  jarl zone serials adbraun.lol --view internal -o json`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			path, qv := zonePath(args[0], view)
+			var resp api.SerialsResponse
+			if err := c.Get(ctx(), path+"/serials", qv, &resp); err != nil {
+				return err
+			}
+			bad := 0
+			for _, r := range resp.Servers {
+				if !r.OK() {
+					bad++
+				}
+			}
+			err = output(resp, func(w io.Writer) {
+				head := displayName(resp.Zone)
+				if resp.View != "" {
+					head += " [view " + resp.View + "]"
+				}
+				head += ", " + resp.Type
+				if resp.Serial != nil {
+					src := map[string]string{"named": "served by named here", "jarl": "Jarl's; named not asked"}[resp.SerialSource]
+					head += fmt.Sprintf(", serial %d (%s)", *resp.Serial, src)
+				}
+				fmt.Fprintf(w, "%s\n\n", head)
+				if len(resp.Servers) > 0 {
+					fmt.Fprintln(w, "SERVER\tADDRESS\tFROM\tSTATUS\tSERIAL\tTIME")
+					for _, r := range resp.Servers {
+						serial, rtt := "-", ""
+						if r.Serial != nil {
+							serial = fmt.Sprint(*r.Serial)
+						}
+						if r.RTTMs > 0 || r.Status == "in_sync" || r.Status == "behind" || r.Status == "ahead" {
+							rtt = fmt.Sprintf("%d ms", r.RTTMs)
+						}
+						addr := r.Address
+						if r.Port != "" {
+							addr += " port " + r.Port
+						}
+						fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", firstNonEmpty(displayName(r.Name), "-"), firstNonEmpty(addr, "-"), r.Source, nsStatusText(r), serial, rtt)
+					}
+				}
+				flush(w)
+				if len(resp.Notes) > 0 {
+					fmt.Println()
+				}
+				for _, n := range resp.Notes {
+					fmt.Printf("note: %s\n", n)
+				}
+			})
+			if err != nil {
+				return err
+			}
+			if bad > 0 {
+				if bad == 1 {
+					return &checkFailed{fmt.Sprintf("1 of %d name servers isn't serving the current serial", len(resp.Servers))}
+				}
+				return &checkFailed{fmt.Sprintf("%d of %d name servers aren't serving the current serial", bad, len(resp.Servers))}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&view, "view", "", "view of the zone")
+	return cmd
+}
+
+func zoneDelegationCmd() *cobra.Command {
+	var view, resolver string
+	cmd := &cobra.Command{
+		Use:   "delegation ZONE",
+		Short: "Check how the parent zone delegates the zone (for zones on the internet)",
+		Long: `Checks the zone the way the internet sees it: finds the parent zone (for
+adbraun.lol, "lol."), asks each of its servers for the zone's delegation,
+and compares the name servers and glue addresses it hands out with the
+zone's NS and address records in Jarl. Then asks each delegated name server
+for the zone's serial.
+
+A zone used only on your own network isn't delegated; that's reported as a
+note, not a problem. Nothing runs this check unless you ask for it.
+
+The parent is found through jarld's resolver (/etc/resolv.conf on the
+server), or --resolver. Exits with 2 when the check finds an error (such as
+a name server that doesn't serve the zone, or missing glue); warnings and
+notes exit with 0.`,
+		Example: `  jarl zone delegation adbraun.lol --view external
+  jarl zone delegation adbraun.lol --resolver 9.9.9.9`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, err := newClient()
+			if err != nil {
+				return err
+			}
+			path, qv := zonePath(args[0], view)
+			if resolver != "" {
+				qv.Set("resolver", resolver)
+			}
+			var resp api.DelegationResponse
+			if err := c.Get(ctx(), path+"/delegation", qv, &resp); err != nil {
+				return err
+			}
+			errors := 0
+			for _, f := range resp.Findings {
+				if f.Level == "error" {
+					errors++
+				}
+			}
+			err = output(resp, func(w io.Writer) {
+				rep := resp.DelegationReport
+				fmt.Fprintf(w, "Zone:\t%s\n", displayName(rep.Zone))
+				if rep.Parent != "" {
+					fmt.Fprintf(w, "Parent zone:\t%s (found through %s)\n", rep.Parent, rep.Resolver)
+				}
+				// The parent's servers, grouped by what they say (the root
+				// zone alone has 13).
+				var order []string
+				groups := map[string][]string{}
+				for _, p := range rep.Parents {
+					var say string
+					switch p.Status {
+					case "delegated":
+						say = "delegate to " + strings.Join(displayNames(p.NS), ", ")
+					case "not_delegated":
+						say = "no delegation"
+					default:
+						say = p.Status + ": " + p.Error
+					}
+					if groups[say] == nil {
+						order = append(order, say)
+					}
+					groups[say] = append(groups[say], displayName(p.Name))
+				}
+				for _, say := range order {
+					who := groups[say][0]
+					if n := len(groups[say]); n > 1 {
+						who = fmt.Sprintf("%s and %d more", who, n-1)
+					}
+					fmt.Fprintf(w, "  %s:\t%s\n", who, say)
+				}
+				if rep.Delegated {
+					fmt.Fprintf(w, "NS at the parent:\t%s\n", strings.Join(displayNames(rep.ParentNS), ", "))
+					fmt.Fprintf(w, "NS in the zone:\t%s\n", strings.Join(displayNames(rep.JarlNS), ", "))
+					for _, g := range rep.Glue {
+						mark := "matches"
+						if !g.Match {
+							mark = "zone says " + firstNonEmpty(strings.Join(g.Jarl, ", "), "nothing")
+						}
+						fmt.Fprintf(w, "Glue %s:\t%s (%s)\n", displayName(g.Name), firstNonEmpty(strings.Join(g.Parent, ", "), "none"), mark)
+					}
+				}
+				flush(w)
+				if len(rep.Servers) > 0 {
+					fmt.Fprintln(w, "\nSERVER\tADDRESS\tSTATUS\tSERIAL")
+					for _, r := range rep.Servers {
+						serial := "-"
+						if r.Serial != nil {
+							serial = fmt.Sprint(*r.Serial)
+						}
+						fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", displayName(r.Name), firstNonEmpty(r.Address, "-"), nsStatusText(r), serial)
+					}
+					flush(w)
+				}
+				fmt.Println()
+				for _, f := range rep.Findings {
+					fmt.Printf("%-8s %s\n", f.Level+":", f.Message)
+				}
+				for _, n := range resp.Notes {
+					fmt.Printf("note:    %s\n", n)
+				}
+			})
+			if err != nil {
+				return err
+			}
+			if errors > 0 {
+				return &checkFailed{fmt.Sprintf("the delegation check found %d error(s)", errors)}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&view, "view", "", "view of the zone")
+	cmd.Flags().StringVar(&resolver, "resolver", "", "resolver to find the parent zone with, e.g. 9.9.9.9 (default: the server's)")
+	return cmd
+}
+
+// nsStatusText describes a name server's answer in a few words.
+func nsStatusText(r nscheck.Result) string {
+	switch r.Status {
+	case "self":
+		return "this server"
+	case "in_sync":
+		return "in sync"
+	case "answered":
+		return "answers"
+	case "behind":
+		return fmt.Sprintf("BEHIND by %d", -r.Diff)
+	case "ahead":
+		return fmt.Sprintf("AHEAD by %d", r.Diff)
+	case "not_authoritative":
+		return "DOESN'T SERVE THE ZONE"
+	case "refused":
+		return "REFUSED"
+	case "unreachable":
+		return "NO ANSWER"
+	case "no_route":
+		return "no route from here"
+	case "no_address":
+		return "NO ADDRESS: " + r.Error
+	}
+	return strings.ToUpper(r.Status) + ": " + r.Error
+}
+
+func displayName(n string) string {
+	if n == "" || n == "." {
+		return n
+	}
+	return strings.TrimSuffix(n, ".")
+}
+
+func displayNames(ns []string) []string {
+	out := make([]string, len(ns))
+	for i, n := range ns {
+		out[i] = displayName(n)
+	}
+	return out
+}
+
+func flush(w io.Writer) {
+	if f, ok := w.(interface{ Flush() error }); ok {
+		f.Flush()
+	}
+}

cmd/jarl/zones.go

@@ -367,7 +367,7 @@ equal to the file: records not in the file are deleted.`,
 	importCmd.Flags().BoolVar(&replace, "replace", false, "delete records that are not in the file")
 	importCmd.Flags().BoolVar(&useSOA, "use-soa", false, "also take SOA timers and $TTL from the file")
 
-	cmd.AddCommand(list, show, create, update, del, fileCmd, importCmd)
+	cmd.AddCommand(list, show, create, update, del, fileCmd, importCmd, zoneSerialsCmd(), zoneDelegationCmd())
 	return cmd
 }
 

cmd/jarld/admin.go

@@ -356,30 +356,6 @@ func resyncCmd() *cobra.Command {
 	return cmd
 }
 
-func backupCmd() *cobra.Command {
-	return &cobra.Command{
-		Use:   "backup FILE",
-		Short: "Write a consistent copy of the database to FILE (safe while jarld runs)",
-		Args:  cobra.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			a, err := openApp(true)
-			if err != nil {
-				return err
-			}
-			defer a.close()
-			if _, err := os.Stat(args[0]); err == nil {
-				return fmt.Errorf("%s already exists", args[0])
-			}
-			if err := a.store.Backup(context.Background(), args[0]); err != nil {
-				return err
-			}
-			os.Chmod(args[0], 0o600)
-			fmt.Println("Database copied to", args[0])
-			return nil
-		},
-	}
-}
-
 func configCmd() *cobra.Command {
 	var example bool
 	cmd := &cobra.Command{

cmd/jarld/backup.go

@@ -0,0 +1,374 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+
+	"jarl/internal/auth"
+	"jarl/internal/backup"
+	"jarl/internal/config"
+	"jarl/internal/store"
+)
+
+func backupCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "backup [FILE]",
+		Short: "Back up the database (safe while jarld runs)",
+		Long: `Writes a consistent copy of Jarl's database and checks it.
+
+Without FILE the copy goes into backup.dir from jarl.yaml (/var/backups/jarl),
+named jarl-<UTC time>.db (plus stats-<UTC time>.db with backup.stats), and
+older backups there are deleted as backup.retention and backup.keep_at_least
+say. This is what the jarl-backup.timer systemd timer runs every day.
+
+With FILE it writes a single copy there, which is never deleted
+automatically.
+
+Every backup is recorded in the database, and the dashboard shows the newest.
+Restore one with 'jarld restore'.`,
+		Example: `  sudo -u jarl jarld backup
+  sudo -u jarl jarld backup /root/jarl-before-upgrade.db`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			if _, err := os.Stat(cfg.Database); err != nil {
+				return fmt.Errorf("no database at %s to back up (create it with 'jarld init')", cfg.Database)
+			}
+			// Not store.Open: a backup must never create or upgrade the
+			// database a running jarld uses.
+			st, err := store.OpenExisting(cfg.Database)
+			if err != nil {
+				return fmt.Errorf("opening %s: %w", cfg.Database, err)
+			}
+			defer st.Close()
+			// Run as root, leave no root-owned files behind for jarld.
+			defer newDirOwner(filepath.Dir(cfg.Database)).fix(filepath.Dir(cfg.Database))
+			ctx := context.Background()
+			now := time.Now()
+			fail := func(err error) error {
+				if rerr := st.RecordBackup(ctx, store.BackupRecord{At: now, Error: err.Error()}); rerr != nil {
+					fmt.Fprintln(os.Stderr, "warning: couldn't record the failure in the database:", rerr)
+				}
+				return err
+			}
+			if len(args) == 1 {
+				dst := args[0]
+				if _, err := os.Lstat(dst); err == nil {
+					return fmt.Errorf("%s already exists", dst)
+				}
+				if err := backup.Copy(ctx, cfg.Database, dst); err != nil {
+					os.Remove(dst)
+					return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
+				}
+				if err := backup.Verify(ctx, dst); err != nil {
+					os.Remove(dst)
+					return fail(fmt.Errorf("the backup failed its check: %w", err))
+				}
+				fi, _ := os.Stat(dst)
+				st.RecordBackup(ctx, store.BackupRecord{At: now, File: dst, Size: fi.Size()})
+				fmt.Printf("Database backed up to %s (%s)\n", dst, size(fi.Size()))
+				return nil
+			}
+
+			dir := cfg.Backup.Dir
+			defer newDirOwner(dir).fix(dir)
+			f, err := backup.Write(ctx, cfg.Database, dir, backup.KindJarl, now)
+			if err != nil {
+				return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
+			}
+			fmt.Printf("Database backed up to %s (%s)\n", f.Path, size(f.Size))
+			kinds := []string{backup.KindJarl}
+			if cfg.Backup.Stats {
+				if _, err := os.Stat(cfg.Stats.Database); err == nil {
+					sf, err := backup.Write(ctx, cfg.Stats.Database, dir, backup.KindStats, now)
+					if err != nil {
+						return fail(fmt.Errorf("backing up the statistics database %s: %w", cfg.Stats.Database, err))
+					}
+					fmt.Printf("Statistics backed up to %s (%s)\n", sf.Path, size(sf.Size))
+				}
+				kinds = append(kinds, backup.KindStats)
+			}
+			for _, kind := range kinds {
+				removed, err := backup.Prune(dir, kind, now, time.Duration(cfg.Backup.Retention), cfg.Backup.KeepAtLeast)
+				for _, r := range removed {
+					fmt.Printf("Deleted %s (older than %s)\n", r.Path, fmtDuration(time.Duration(cfg.Backup.Retention)))
+				}
+				if err != nil {
+					return fail(fmt.Errorf("deleting old backups: %w", err))
+				}
+			}
+			if err := st.RecordBackup(ctx, store.BackupRecord{At: now, File: f.Path, Size: f.Size}); err != nil {
+				fmt.Fprintln(os.Stderr, "warning: couldn't record the backup in the database:", err)
+			}
+			return nil
+		},
+	}
+}
+
+func restoreCmd() *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use:   "restore [FILE | latest]",
+		Short: "Replace the database with a backup (jarld must be stopped)",
+		Long: `Without arguments, lists the backups in backup.dir.
+
+With FILE, or "latest" for the newest backup in backup.dir, it checks the
+backup, shows what it holds next to the current database, and after you
+confirm replaces the database with it. jarld must be stopped first. The
+current database is kept as DATABASE.before-restore-<UTC time>, and the
+restore is recorded in the restored database's audit log.
+
+Then start jarld and bring named's files in line with the restored database:
+
+  systemctl start jarl
+  jarl resync --dry-run     # what differs
+  jarl resync --force       # write everything and reload named
+
+See "Restoring" in docs/operations.md, also for zone serials and secondaries.`,
+		Example: `  sudo jarld restore
+  sudo systemctl stop jarl
+  sudo jarld restore latest
+  sudo jarld restore /var/backups/jarl/jarl-20261008T031512Z.db`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			ctx := context.Background()
+			if len(args) == 0 {
+				return listBackups(cfg.Backup.Dir)
+			}
+			src := args[0]
+			if src == "latest" {
+				files, err := backup.List(cfg.Backup.Dir)
+				if err != nil {
+					return err
+				}
+				src = ""
+				for _, f := range files {
+					if f.Kind == backup.KindJarl {
+						src = f.Path
+						break
+					}
+				}
+				if src == "" {
+					return fmt.Errorf("no backups in %s", cfg.Backup.Dir)
+				}
+			}
+			if err := backup.Verify(ctx, src); err != nil {
+				return fmt.Errorf("%s can't be restored: %w", src, err)
+			}
+			info, err := backup.Inspect(ctx, src)
+			if err != nil {
+				return err
+			}
+			if info.Schema > store.SchemaVersion() {
+				return fmt.Errorf("%s was made by a newer Jarl (schema %d, this one knows %d); install that version to restore it", src, info.Schema, store.SchemaVersion())
+			}
+			unlock, err := lockDatabase(cfg.Database)
+			if errors.Is(err, errInUse) {
+				return fmt.Errorf("jarld is running; stop it first (systemctl stop jarl), then run this again")
+			}
+			if err != nil {
+				return err
+			}
+			defer unlock()
+
+			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+			fmt.Fprintf(w, "Backup:\t%s\n", src)
+			describe(w, info, backupTime(src))
+			if cur, err := backup.Inspect(ctx, cfg.Database); err == nil {
+				fmt.Fprintf(w, "\nCurrent database:\t%s\n", cfg.Database)
+				describe(w, cur, time.Time{})
+			} else if !errors.Is(err, os.ErrNotExist) {
+				fmt.Fprintf(w, "\nCurrent database:\t%s (unreadable: %v)\n", cfg.Database, err)
+			}
+			w.Flush()
+			if !yes {
+				if !term.IsTerminal(int(os.Stdin.Fd())) {
+					return fmt.Errorf("add --yes to restore without being asked")
+				}
+				fmt.Print("\nReplace the current database with this backup? [y/N] ")
+				line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
+				if a := strings.ToLower(strings.TrimSpace(line)); a != "y" && a != "yes" {
+					return fmt.Errorf("nothing changed")
+				}
+			}
+
+			db := cfg.Database
+			defer newDirOwner(filepath.Dir(db)).fix(filepath.Dir(db))
+			aside := db + ".before-restore-" + time.Now().UTC().Format("20060102T150405Z")
+			// The database's WAL and shared-memory files belong to it:
+			// they move with it, and the restored copy starts without.
+			moved := false
+			for _, suffix := range []string{"", "-wal", "-shm"} {
+				if _, err := os.Lstat(db + suffix); err == nil {
+					if err := os.Rename(db+suffix, aside+suffix); err != nil {
+						return err
+					}
+					moved = true
+				}
+			}
+			if err := copyFile(src, db, 0o640); err != nil {
+				if moved {
+					os.Rename(aside, db)
+					os.Rename(aside+"-wal", db+"-wal")
+					os.Rename(aside+"-shm", db+"-shm")
+				}
+				return fmt.Errorf("restoring: %w (the current database was left in place)", err)
+			}
+			// Open it as jarld will: an older backup gets the current
+			// schema now, and the restore goes into its audit log.
+			a, err := openApp(true)
+			if err != nil {
+				return fmt.Errorf("the backup was copied to %s but doesn't open: %w", db, err)
+			}
+			p := &auth.Principal{Method: "local"}
+			if lp, err := a.localPrincipal(ctx, ""); err == nil {
+				p = lp
+			}
+			a.eng.Audit(ctx, p, "system.restore", "", 0, "", "success", "database restored from "+src)
+			a.close()
+
+			fmt.Printf("\nRestored %s from %s.\n", db, src)
+			if moved {
+				fmt.Printf("The previous database is in %s.\n", aside)
+			}
+			fmt.Print(`
+Next:
+  systemctl start jarl
+  jarl resync --dry-run     # what differs between the restored database and named's files
+  jarl resync --force       # write them and reload named
+
+If you have secondary servers, zone serials older than what they last saw
+are ignored by them: raise them with 'jarl zone update ZONE --serial N'
+(see "Restoring" in docs/operations.md).
+`)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask for confirmation")
+	return cmd
+}
+
+func listBackups(dir string) error {
+	files, err := backup.List(dir)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		fmt.Printf("No backups in %s.\n", dir)
+		return nil
+	}
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "MADE\tAGE\tKIND\tSIZE\tFILE")
+	for _, f := range files {
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", f.Time.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(f.Time)), f.Kind, size(f.Size), f.Path)
+	}
+	w.Flush()
+	fmt.Println("\nRestore one with: jarld restore FILE   (or: jarld restore latest)")
+	return nil
+}
+
+func describe(w io.Writer, in *backup.Info, made time.Time) {
+	if !made.IsZero() {
+		fmt.Fprintf(w, "  made\t%s (%s ago)\n", made.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(made)))
+	}
+	fmt.Fprintf(w, "  contents\t%s, %s, %s (schema %d)\n", plural(in.Zones, "zone"), plural(in.Records, "record"), plural(in.Users, "user"), in.Schema)
+	if in.LastChange != nil {
+		fmt.Fprintf(w, "  last change\t%s\n", in.LastChange.Local().Format("2006-01-02 15:04:05"))
+	}
+}
+
+// backupTime is when a backup was made: from its name if it has a
+// backup's name, otherwise its modification time.
+func backupTime(path string) time.Time {
+	for _, f := range mustList(filepath.Dir(path)) {
+		if f.Path == filepath.Clean(path) {
+			return f.Time
+		}
+	}
+	if fi, err := os.Stat(path); err == nil {
+		return fi.ModTime()
+	}
+	return time.Time{}
+}
+
+func mustList(dir string) []backup.File {
+	files, _ := backup.List(dir)
+	return files
+}
+
+// copyFile copies src to dst through a temporary file, so dst is either
+// complete or absent.
+func copyFile(src, dst string, mode os.FileMode) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	tmp := dst + ".restore-tmp"
+	out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := out.Sync(); err != nil {
+		out.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := out.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, dst)
+}
+
+func size(n int64) string {
+	switch {
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
+	}
+	return fmt.Sprintf("%d bytes", n)
+}
+
+func fmtDuration(d time.Duration) string {
+	switch {
+	case d >= 48*time.Hour:
+		return fmt.Sprintf("%d days", int(d/(24*time.Hour)))
+	case d >= 2*time.Hour:
+		return fmt.Sprintf("%d hours", int(d/time.Hour))
+	case d >= 2*time.Minute:
+		return fmt.Sprintf("%d minutes", int(d/time.Minute))
+	}
+	return "under 2 minutes"
+}
+
+func plural(n int, word string) string {
+	if n == 1 {
+		return "1 " + word
+	}
+	return fmt.Sprintf("%d %ss", n, word)
+}

cmd/jarld/lock.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// errInUse means another process holds the database lock.
+var errInUse = errors.New("the database is in use by a running jarld or by 'jarld restore'")
+
+// lockDatabase takes the exclusive lock (on DATABASE.lock) that marks the
+// database as in use by a running jarld or by 'jarld restore', so a
+// restore can never replace the database under a running jarld, and two
+// jarlds can't serve the same database. Other commands (backup, user, ...)
+// don't take it: they work alongside a running jarld.
+func lockDatabase(db string) (release func(), err error) {
+	if err := os.MkdirAll(filepath.Dir(db), 0o750); err != nil {
+		return nil, err
+	}
+	path := db + ".lock"
+	// Read-only is enough for flock, and works whoever created the file.
+	f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0o640)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		f.Close()
+		if errors.Is(err, syscall.EWOULDBLOCK) {
+			return nil, errInUse
+		}
+		return nil, fmt.Errorf("locking %s: %w", path, err)
+	}
+	return func() {
+		syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
+}

cmd/jarld/main.go

@@ -57,7 +57,7 @@ func rootCmd() *cobra.Command {
 		def = config.DefaultPath
 	}
 	root.PersistentFlags().StringVarP(&configPath, "config", "c", def, "configuration file")
-	root.AddCommand(serveCmd(), initCmd(), userCmd(), importCmd(), adoptCmd(), checkCmd(), renderCmd(), resyncCmd(), backupCmd(), configCmd(), versionCmd())
+	root.AddCommand(serveCmd(), initCmd(), userCmd(), importCmd(), adoptCmd(), checkCmd(), renderCmd(), resyncCmd(), backupCmd(), restoreCmd(), configCmd(), versionCmd())
 	return root
 }
 
@@ -226,6 +226,15 @@ func serveCmd() *cobra.Command {
 		Use:   "serve",
 		Short: "Run the daemon (web UI and API)",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			unlock, err := lockDatabase(cfg.Database)
+			if err != nil {
+				return fmt.Errorf("%s: %w", cfg.Database, err)
+			}
+			defer unlock()
 			a, err := openApp(true)
 			if err != nil {
 				return err
@@ -287,6 +296,7 @@ func serveCmd() *cobra.Command {
 				TrustedProxies: a.cfg.Proxies(), SessionLifetime: time.Duration(a.cfg.Session.Lifetime),
 				SessionIdle: time.Duration(a.cfg.Session.IdleTimeout), CookieSecure: a.cfg.Session.CookieSecure,
 				Version: version.Version, Stats: collector, Socket: a.cfg.Socket,
+				BackupDir: a.cfg.Backup.Dir, BackupWarnAfter: time.Duration(a.cfg.Backup.WarnAfter),
 				HTTPReq: server.HTTPReq{
 					Enabled:           a.cfg.ACME.HTTPReq.Enabled,
 					AllowPlainHTTP:    !a.cfg.ACME.HTTPReq.RequireHTTPS,