Add backups, token expiry warnings and name server checks

- Daily database backups: jarl-backup.timer runs 'jarld backup', which
  writes jarl-<UTC time>.db into backup.dir, checks it before naming it,
  and prunes by the time in the name while always keeping the newest
  keep_at_least. New backup: section in jarl.yaml; the dashboard and
  'jarl status' show the newest backup and warn when it is stale or
  failed. The Debian package enables the timer.
- 'jarld restore [FILE | latest]': checks the backup, shows it next to
  the current database, keeps the current one aside and records the
  restore. 'jarld serve' holds a database lock, so a restore can't run
  under a live jarld and two jarlds can't share a database.
- Token expiry warnings from 14 days ahead: dashboard, token lists, a
  daily log line, and a CLI warning via the Jarl-Token-Expires header;
  'jarl token list --expiring'.
- 'jarl zone serials' and the zone page's Name servers tab: compare the
  serial on a primary zone's secondaries (NS records, also-notify) or a
  secondary zone's primaries with this server's.
- 'jarl zone delegation': for zones on the internet, check the parent's
  delegation and glue against the zone; an undelegated zone gets a note.
- Server: Jarl/<version> header; CLI exit code 2 for failed checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/jarld/admin.go

@@ -356,30 +356,6 @@ func resyncCmd() *cobra.Command {
 	return cmd
 }
 
-func backupCmd() *cobra.Command {
-	return &cobra.Command{
-		Use:   "backup FILE",
-		Short: "Write a consistent copy of the database to FILE (safe while jarld runs)",
-		Args:  cobra.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			a, err := openApp(true)
-			if err != nil {
-				return err
-			}
-			defer a.close()
-			if _, err := os.Stat(args[0]); err == nil {
-				return fmt.Errorf("%s already exists", args[0])
-			}
-			if err := a.store.Backup(context.Background(), args[0]); err != nil {
-				return err
-			}
-			os.Chmod(args[0], 0o600)
-			fmt.Println("Database copied to", args[0])
-			return nil
-		},
-	}
-}
-
 func configCmd() *cobra.Command {
 	var example bool
 	cmd := &cobra.Command{

cmd/jarld/backup.go

@@ -0,0 +1,374 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+
+	"jarl/internal/auth"
+	"jarl/internal/backup"
+	"jarl/internal/config"
+	"jarl/internal/store"
+)
+
+func backupCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "backup [FILE]",
+		Short: "Back up the database (safe while jarld runs)",
+		Long: `Writes a consistent copy of Jarl's database and checks it.
+
+Without FILE the copy goes into backup.dir from jarl.yaml (/var/backups/jarl),
+named jarl-<UTC time>.db (plus stats-<UTC time>.db with backup.stats), and
+older backups there are deleted as backup.retention and backup.keep_at_least
+say. This is what the jarl-backup.timer systemd timer runs every day.
+
+With FILE it writes a single copy there, which is never deleted
+automatically.
+
+Every backup is recorded in the database, and the dashboard shows the newest.
+Restore one with 'jarld restore'.`,
+		Example: `  sudo -u jarl jarld backup
+  sudo -u jarl jarld backup /root/jarl-before-upgrade.db`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			if _, err := os.Stat(cfg.Database); err != nil {
+				return fmt.Errorf("no database at %s to back up (create it with 'jarld init')", cfg.Database)
+			}
+			// Not store.Open: a backup must never create or upgrade the
+			// database a running jarld uses.
+			st, err := store.OpenExisting(cfg.Database)
+			if err != nil {
+				return fmt.Errorf("opening %s: %w", cfg.Database, err)
+			}
+			defer st.Close()
+			// Run as root, leave no root-owned files behind for jarld.
+			defer newDirOwner(filepath.Dir(cfg.Database)).fix(filepath.Dir(cfg.Database))
+			ctx := context.Background()
+			now := time.Now()
+			fail := func(err error) error {
+				if rerr := st.RecordBackup(ctx, store.BackupRecord{At: now, Error: err.Error()}); rerr != nil {
+					fmt.Fprintln(os.Stderr, "warning: couldn't record the failure in the database:", rerr)
+				}
+				return err
+			}
+			if len(args) == 1 {
+				dst := args[0]
+				if _, err := os.Lstat(dst); err == nil {
+					return fmt.Errorf("%s already exists", dst)
+				}
+				if err := backup.Copy(ctx, cfg.Database, dst); err != nil {
+					os.Remove(dst)
+					return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
+				}
+				if err := backup.Verify(ctx, dst); err != nil {
+					os.Remove(dst)
+					return fail(fmt.Errorf("the backup failed its check: %w", err))
+				}
+				fi, _ := os.Stat(dst)
+				st.RecordBackup(ctx, store.BackupRecord{At: now, File: dst, Size: fi.Size()})
+				fmt.Printf("Database backed up to %s (%s)\n", dst, size(fi.Size()))
+				return nil
+			}
+
+			dir := cfg.Backup.Dir
+			defer newDirOwner(dir).fix(dir)
+			f, err := backup.Write(ctx, cfg.Database, dir, backup.KindJarl, now)
+			if err != nil {
+				return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
+			}
+			fmt.Printf("Database backed up to %s (%s)\n", f.Path, size(f.Size))
+			kinds := []string{backup.KindJarl}
+			if cfg.Backup.Stats {
+				if _, err := os.Stat(cfg.Stats.Database); err == nil {
+					sf, err := backup.Write(ctx, cfg.Stats.Database, dir, backup.KindStats, now)
+					if err != nil {
+						return fail(fmt.Errorf("backing up the statistics database %s: %w", cfg.Stats.Database, err))
+					}
+					fmt.Printf("Statistics backed up to %s (%s)\n", sf.Path, size(sf.Size))
+				}
+				kinds = append(kinds, backup.KindStats)
+			}
+			for _, kind := range kinds {
+				removed, err := backup.Prune(dir, kind, now, time.Duration(cfg.Backup.Retention), cfg.Backup.KeepAtLeast)
+				for _, r := range removed {
+					fmt.Printf("Deleted %s (older than %s)\n", r.Path, fmtDuration(time.Duration(cfg.Backup.Retention)))
+				}
+				if err != nil {
+					return fail(fmt.Errorf("deleting old backups: %w", err))
+				}
+			}
+			if err := st.RecordBackup(ctx, store.BackupRecord{At: now, File: f.Path, Size: f.Size}); err != nil {
+				fmt.Fprintln(os.Stderr, "warning: couldn't record the backup in the database:", err)
+			}
+			return nil
+		},
+	}
+}
+
+func restoreCmd() *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use:   "restore [FILE | latest]",
+		Short: "Replace the database with a backup (jarld must be stopped)",
+		Long: `Without arguments, lists the backups in backup.dir.
+
+With FILE, or "latest" for the newest backup in backup.dir, it checks the
+backup, shows what it holds next to the current database, and after you
+confirm replaces the database with it. jarld must be stopped first. The
+current database is kept as DATABASE.before-restore-<UTC time>, and the
+restore is recorded in the restored database's audit log.
+
+Then start jarld and bring named's files in line with the restored database:
+
+  systemctl start jarl
+  jarl resync --dry-run     # what differs
+  jarl resync --force       # write everything and reload named
+
+See "Restoring" in docs/operations.md, also for zone serials and secondaries.`,
+		Example: `  sudo jarld restore
+  sudo systemctl stop jarl
+  sudo jarld restore latest
+  sudo jarld restore /var/backups/jarl/jarl-20261008T031512Z.db`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			ctx := context.Background()
+			if len(args) == 0 {
+				return listBackups(cfg.Backup.Dir)
+			}
+			src := args[0]
+			if src == "latest" {
+				files, err := backup.List(cfg.Backup.Dir)
+				if err != nil {
+					return err
+				}
+				src = ""
+				for _, f := range files {
+					if f.Kind == backup.KindJarl {
+						src = f.Path
+						break
+					}
+				}
+				if src == "" {
+					return fmt.Errorf("no backups in %s", cfg.Backup.Dir)
+				}
+			}
+			if err := backup.Verify(ctx, src); err != nil {
+				return fmt.Errorf("%s can't be restored: %w", src, err)
+			}
+			info, err := backup.Inspect(ctx, src)
+			if err != nil {
+				return err
+			}
+			if info.Schema > store.SchemaVersion() {
+				return fmt.Errorf("%s was made by a newer Jarl (schema %d, this one knows %d); install that version to restore it", src, info.Schema, store.SchemaVersion())
+			}
+			unlock, err := lockDatabase(cfg.Database)
+			if errors.Is(err, errInUse) {
+				return fmt.Errorf("jarld is running; stop it first (systemctl stop jarl), then run this again")
+			}
+			if err != nil {
+				return err
+			}
+			defer unlock()
+
+			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+			fmt.Fprintf(w, "Backup:\t%s\n", src)
+			describe(w, info, backupTime(src))
+			if cur, err := backup.Inspect(ctx, cfg.Database); err == nil {
+				fmt.Fprintf(w, "\nCurrent database:\t%s\n", cfg.Database)
+				describe(w, cur, time.Time{})
+			} else if !errors.Is(err, os.ErrNotExist) {
+				fmt.Fprintf(w, "\nCurrent database:\t%s (unreadable: %v)\n", cfg.Database, err)
+			}
+			w.Flush()
+			if !yes {
+				if !term.IsTerminal(int(os.Stdin.Fd())) {
+					return fmt.Errorf("add --yes to restore without being asked")
+				}
+				fmt.Print("\nReplace the current database with this backup? [y/N] ")
+				line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
+				if a := strings.ToLower(strings.TrimSpace(line)); a != "y" && a != "yes" {
+					return fmt.Errorf("nothing changed")
+				}
+			}
+
+			db := cfg.Database
+			defer newDirOwner(filepath.Dir(db)).fix(filepath.Dir(db))
+			aside := db + ".before-restore-" + time.Now().UTC().Format("20060102T150405Z")
+			// The database's WAL and shared-memory files belong to it:
+			// they move with it, and the restored copy starts without.
+			moved := false
+			for _, suffix := range []string{"", "-wal", "-shm"} {
+				if _, err := os.Lstat(db + suffix); err == nil {
+					if err := os.Rename(db+suffix, aside+suffix); err != nil {
+						return err
+					}
+					moved = true
+				}
+			}
+			if err := copyFile(src, db, 0o640); err != nil {
+				if moved {
+					os.Rename(aside, db)
+					os.Rename(aside+"-wal", db+"-wal")
+					os.Rename(aside+"-shm", db+"-shm")
+				}
+				return fmt.Errorf("restoring: %w (the current database was left in place)", err)
+			}
+			// Open it as jarld will: an older backup gets the current
+			// schema now, and the restore goes into its audit log.
+			a, err := openApp(true)
+			if err != nil {
+				return fmt.Errorf("the backup was copied to %s but doesn't open: %w", db, err)
+			}
+			p := &auth.Principal{Method: "local"}
+			if lp, err := a.localPrincipal(ctx, ""); err == nil {
+				p = lp
+			}
+			a.eng.Audit(ctx, p, "system.restore", "", 0, "", "success", "database restored from "+src)
+			a.close()
+
+			fmt.Printf("\nRestored %s from %s.\n", db, src)
+			if moved {
+				fmt.Printf("The previous database is in %s.\n", aside)
+			}
+			fmt.Print(`
+Next:
+  systemctl start jarl
+  jarl resync --dry-run     # what differs between the restored database and named's files
+  jarl resync --force       # write them and reload named
+
+If you have secondary servers, zone serials older than what they last saw
+are ignored by them: raise them with 'jarl zone update ZONE --serial N'
+(see "Restoring" in docs/operations.md).
+`)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask for confirmation")
+	return cmd
+}
+
+func listBackups(dir string) error {
+	files, err := backup.List(dir)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		fmt.Printf("No backups in %s.\n", dir)
+		return nil
+	}
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "MADE\tAGE\tKIND\tSIZE\tFILE")
+	for _, f := range files {
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", f.Time.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(f.Time)), f.Kind, size(f.Size), f.Path)
+	}
+	w.Flush()
+	fmt.Println("\nRestore one with: jarld restore FILE   (or: jarld restore latest)")
+	return nil
+}
+
+func describe(w io.Writer, in *backup.Info, made time.Time) {
+	if !made.IsZero() {
+		fmt.Fprintf(w, "  made\t%s (%s ago)\n", made.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(made)))
+	}
+	fmt.Fprintf(w, "  contents\t%s, %s, %s (schema %d)\n", plural(in.Zones, "zone"), plural(in.Records, "record"), plural(in.Users, "user"), in.Schema)
+	if in.LastChange != nil {
+		fmt.Fprintf(w, "  last change\t%s\n", in.LastChange.Local().Format("2006-01-02 15:04:05"))
+	}
+}
+
+// backupTime is when a backup was made: from its name if it has a
+// backup's name, otherwise its modification time.
+func backupTime(path string) time.Time {
+	for _, f := range mustList(filepath.Dir(path)) {
+		if f.Path == filepath.Clean(path) {
+			return f.Time
+		}
+	}
+	if fi, err := os.Stat(path); err == nil {
+		return fi.ModTime()
+	}
+	return time.Time{}
+}
+
+func mustList(dir string) []backup.File {
+	files, _ := backup.List(dir)
+	return files
+}
+
+// copyFile copies src to dst through a temporary file, so dst is either
+// complete or absent.
+func copyFile(src, dst string, mode os.FileMode) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	tmp := dst + ".restore-tmp"
+	out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := out.Sync(); err != nil {
+		out.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := out.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, dst)
+}
+
+func size(n int64) string {
+	switch {
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
+	}
+	return fmt.Sprintf("%d bytes", n)
+}
+
+func fmtDuration(d time.Duration) string {
+	switch {
+	case d >= 48*time.Hour:
+		return fmt.Sprintf("%d days", int(d/(24*time.Hour)))
+	case d >= 2*time.Hour:
+		return fmt.Sprintf("%d hours", int(d/time.Hour))
+	case d >= 2*time.Minute:
+		return fmt.Sprintf("%d minutes", int(d/time.Minute))
+	}
+	return "under 2 minutes"
+}
+
+func plural(n int, word string) string {
+	if n == 1 {
+		return "1 " + word
+	}
+	return fmt.Sprintf("%d %ss", n, word)
+}

cmd/jarld/lock.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// errInUse means another process holds the database lock.
+var errInUse = errors.New("the database is in use by a running jarld or by 'jarld restore'")
+
+// lockDatabase takes the exclusive lock (on DATABASE.lock) that marks the
+// database as in use by a running jarld or by 'jarld restore', so a
+// restore can never replace the database under a running jarld, and two
+// jarlds can't serve the same database. Other commands (backup, user, ...)
+// don't take it: they work alongside a running jarld.
+func lockDatabase(db string) (release func(), err error) {
+	if err := os.MkdirAll(filepath.Dir(db), 0o750); err != nil {
+		return nil, err
+	}
+	path := db + ".lock"
+	// Read-only is enough for flock, and works whoever created the file.
+	f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0o640)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		f.Close()
+		if errors.Is(err, syscall.EWOULDBLOCK) {
+			return nil, errInUse
+		}
+		return nil, fmt.Errorf("locking %s: %w", path, err)
+	}
+	return func() {
+		syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
+}

cmd/jarld/main.go

@@ -57,7 +57,7 @@ func rootCmd() *cobra.Command {
 		def = config.DefaultPath
 	}
 	root.PersistentFlags().StringVarP(&configPath, "config", "c", def, "configuration file")
-	root.AddCommand(serveCmd(), initCmd(), userCmd(), importCmd(), adoptCmd(), checkCmd(), renderCmd(), resyncCmd(), backupCmd(), configCmd(), versionCmd())
+	root.AddCommand(serveCmd(), initCmd(), userCmd(), importCmd(), adoptCmd(), checkCmd(), renderCmd(), resyncCmd(), backupCmd(), restoreCmd(), configCmd(), versionCmd())
 	return root
 }
 
@@ -226,6 +226,15 @@ func serveCmd() *cobra.Command {
 		Use:   "serve",
 		Short: "Run the daemon (web UI and API)",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load(configPath, true)
+			if err != nil {
+				return fmt.Errorf("loading configuration: %w", err)
+			}
+			unlock, err := lockDatabase(cfg.Database)
+			if err != nil {
+				return fmt.Errorf("%s: %w", cfg.Database, err)
+			}
+			defer unlock()
 			a, err := openApp(true)
 			if err != nil {
 				return err
@@ -287,6 +296,7 @@ func serveCmd() *cobra.Command {
 				TrustedProxies: a.cfg.Proxies(), SessionLifetime: time.Duration(a.cfg.Session.Lifetime),
 				SessionIdle: time.Duration(a.cfg.Session.IdleTimeout), CookieSecure: a.cfg.Session.CookieSecure,
 				Version: version.Version, Stats: collector, Socket: a.cfg.Socket,
+				BackupDir: a.cfg.Backup.Dir, BackupWarnAfter: time.Duration(a.cfg.Backup.WarnAfter),
 				HTTPReq: server.HTTPReq{
 					Enabled:           a.cfg.ACME.HTTPReq.Enabled,
 					AllowPlainHTTP:    !a.cfg.ACME.HTTPReq.RequireHTTPS,