jarl/cmd/jarld/backup.go

package main

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/spf13/cobra"
	"golang.org/x/term"

	"jarl/internal/auth"
	"jarl/internal/backup"
	"jarl/internal/config"
	"jarl/internal/render"
	"jarl/internal/store"
)

func backupCmd() *cobra.Command {
	var withoutKeys bool
	cmd := &cobra.Command{
		Use:   "backup [FILE]",
		Short: "Back up the database (safe while jarld runs)",
		Long: `Writes a consistent copy of Jarl's database and checks it.

Without FILE the copy goes into backup.dir from jarl.yaml (/var/backups/jarl),
named jarl-<UTC time>.db (plus stats-<UTC time>.db with backup.stats), and
older backups there are deleted as backup.retention and backup.keep_at_least
say. This is what the jarl-backup.timer systemd timer runs every day.

With FILE it writes a single copy there, which is never deleted
automatically.

Each backup also holds the DNSSEC keys of signed zones (from Jarl's key
directory, private keys included), so that 'jarld restore' can put them
back: a zone whose keys are lost stops validating while its parent still
has a DS record for them. named's private keys are readable only by named's
user, so run this as root; jarl-backup.service has the capability to read
them. --without-keys backs up only the database.

Every backup is recorded in the database, and the dashboard shows the newest.
Restore one with 'jarld restore'.`,
		Example: `  sudo jarld backup
  sudo jarld backup /root/jarl-before-upgrade.db
  sudo systemctl start jarl-backup.service`,
		Args: cobra.MaximumNArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			cfg, err := config.Load(configPath, true)
			if err != nil {
				return fmt.Errorf("loading configuration: %w", err)
			}
			if _, err := os.Stat(cfg.Database); err != nil {
				return fmt.Errorf("no database at %s to back up (create it with 'jarld init')", cfg.Database)
			}
			// Not store.Open: a backup must never create or upgrade the
			// database a running jarld uses.
			st, err := store.OpenExisting(cfg.Database)
			if err != nil {
				return fmt.Errorf("opening %s: %w", cfg.Database, err)
			}
			defer st.Close()
			// Run as root, leave no root-owned files behind for jarld.
			defer newDirOwner(filepath.Dir(cfg.Database)).fix(filepath.Dir(cfg.Database))
			ctx := context.Background()
			now := time.Now()
			keysDir := render.Paths{ZoneDir: cfg.BIND.ZoneDir}.KeysDir()
			if withoutKeys {
				keysDir = ""
			}
			fail := func(err error) error {
				if strings.Contains(err.Error(), "DNSSEC keys") {
					err = fmt.Errorf("%w; or back up only the database with --without-keys", err)
				}
				if rerr := st.RecordBackup(ctx, store.BackupRecord{At: now, Error: err.Error()}); rerr != nil {
					fmt.Fprintln(os.Stderr, "warning: couldn't record the failure in the database:", rerr)
				}
				return err
			}
			if len(args) == 1 {
				dst := args[0]
				if _, err := os.Lstat(dst); err == nil {
					return fmt.Errorf("%s already exists", dst)
				}
				if err := backup.Copy(ctx, cfg.Database, dst); err != nil {
					os.Remove(dst)
					return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
				}
				if keysDir != "" {
					if _, err := backup.AddFiles(ctx, dst, keysDir); err != nil {
						os.Remove(dst)
						return fail(err)
					}
				}
				if err := backup.Verify(ctx, dst); err != nil {
					os.Remove(dst)
					return fail(fmt.Errorf("the backup failed its check: %w", err))
				}
				fi, _ := os.Stat(dst)
				st.RecordBackup(ctx, store.BackupRecord{At: now, File: dst, Size: fi.Size()})
				fmt.Printf("Database backed up to %s (%s)%s\n", dst, size(fi.Size()), keysNote(ctx, dst))
				return nil
			}

			dir := cfg.Backup.Dir
			defer newDirOwner(dir).fix(dir)
			f, err := backup.Write(ctx, cfg.Database, dir, backup.KindJarl, now, keysDir)
			if err != nil {
				return fail(fmt.Errorf("backing up %s: %w", cfg.Database, err))
			}
			fmt.Printf("Database backed up to %s (%s)%s\n", f.Path, size(f.Size), keysNote(ctx, f.Path))
			kinds := []string{backup.KindJarl}
			if cfg.Backup.Stats {
				if _, err := os.Stat(cfg.Stats.Database); err == nil {
					sf, err := backup.Write(ctx, cfg.Stats.Database, dir, backup.KindStats, now, "")
					if err != nil {
						return fail(fmt.Errorf("backing up the statistics database %s: %w", cfg.Stats.Database, err))
					}
					fmt.Printf("Statistics backed up to %s (%s)\n", sf.Path, size(sf.Size))
				}
				kinds = append(kinds, backup.KindStats)
			}
			for _, kind := range kinds {
				removed, err := backup.Prune(dir, kind, now, time.Duration(cfg.Backup.Retention), cfg.Backup.KeepAtLeast)
				for _, r := range removed {
					fmt.Printf("Deleted %s (older than %s)\n", r.Path, fmtDuration(time.Duration(cfg.Backup.Retention)))
				}
				if err != nil {
					return fail(fmt.Errorf("deleting old backups: %w", err))
				}
			}
			if err := st.RecordBackup(ctx, store.BackupRecord{At: now, File: f.Path, Size: f.Size}); err != nil {
				fmt.Fprintln(os.Stderr, "warning: couldn't record the backup in the database:", err)
			}
			return nil
		},
	}
	cmd.Flags().BoolVar(&withoutKeys, "without-keys", false, "back up only the database, not the DNSSEC keys")
	return cmd
}

// keysNote says how many DNSSEC key files a backup holds.
func keysNote(ctx context.Context, path string) string {
	if in, err := backup.Inspect(ctx, path); err == nil && in.KeyFiles > 0 {
		return ", with " + plural(in.KeyFiles, "DNSSEC key file")
	}
	return ""
}

func restoreCmd() *cobra.Command {
	var yes, withoutKeys bool
	cmd := &cobra.Command{
		Use:   "restore [FILE | latest]",
		Short: "Replace the database with a backup (jarld must be stopped)",
		Long: `Without arguments, lists the backups in backup.dir.

With FILE, or "latest" for the newest backup in backup.dir, it checks the
backup, shows what it holds next to the current database, and after you
confirm replaces the database with it. jarld must be stopped first. The
current database is kept as DATABASE.before-restore-<UTC time>, and the
restore is recorded in the restored database's audit log.

The DNSSEC keys stored in the backup go back into Jarl's key directory,
owned by named's user (bind.named_user), which needs root. Key files that
are there but not in the backup are left alone. --without-keys restores
only the database.

Then start jarld and bring named's files in line with the restored database:

  systemctl start jarl
  jarl resync --dry-run     # what differs
  jarl resync --force       # write everything and reload named

Jarl raises the serials of zones whose data goes back to an older state
above what named and the secondaries have, so they pick it up. See
"Restoring" in docs/operations.md.`,
		Example: `  sudo jarld restore
  sudo systemctl stop jarl
  sudo jarld restore latest
  sudo jarld restore /var/backups/jarl/jarl-20261008T031512Z.db`,
		Args: cobra.MaximumNArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			cfg, err := config.Load(configPath, true)
			if err != nil {
				return fmt.Errorf("loading configuration: %w", err)
			}
			ctx := context.Background()
			if len(args) == 0 {
				return listBackups(cfg.Backup.Dir)
			}
			src := args[0]
			if src == "latest" {
				files, err := backup.List(cfg.Backup.Dir)
				if err != nil {
					return err
				}
				src = ""
				for _, f := range files {
					if f.Kind == backup.KindJarl {
						src = f.Path
						break
					}
				}
				if src == "" {
					return fmt.Errorf("no backups in %s", cfg.Backup.Dir)
				}
			}
			if err := backup.Verify(ctx, src); err != nil {
				return fmt.Errorf("%s can't be restored: %w", src, err)
			}
			info, err := backup.Inspect(ctx, src)
			if err != nil {
				return err
			}
			if info.Schema > store.SchemaVersion() {
				return fmt.Errorf("%s was made by a newer Jarl (schema %d, this one knows %d); install that version to restore it", src, info.Schema, store.SchemaVersion())
			}
			keysDir := render.Paths{ZoneDir: cfg.BIND.ZoneDir}.KeysDir()
			var keyFiles []backup.StoredFile
			if !withoutKeys {
				var root string
				root, keyFiles, err = backup.Files(ctx, src)
				if err != nil {
					return fmt.Errorf("reading the DNSSEC keys in %s: %w", src, err)
				}
				if len(keyFiles) > 0 && os.Geteuid() != 0 && !runningAs(cfg.BIND.NamedUser) {
					return fmt.Errorf("the backup holds %s, and putting them back for named needs root: run this as root (or add --without-keys to restore only the database)", plural(len(keyFiles), "DNSSEC key file"))
				}
				if root != "" && root != keysDir && len(keyFiles) > 0 {
					fmt.Printf("The keys were backed up from %s; they go to %s now.\n", root, keysDir)
				}
			}
			if info.SignedZones > 0 && len(keyFiles) == 0 && !withoutKeys {
				fmt.Fprintf(os.Stderr, "warning: the backup has %s but no DNSSEC keys (it was made with --without-keys, or before Jarl backed them up). Unless the keys are still in %s, named makes new ones, and a DS record for the old ones at the parent stops matching.\n", plural(info.SignedZones, "signed zone"), keysDir)
			}
			unlock, err := lockDatabase(cfg.Database)
			if errors.Is(err, errInUse) {
				return fmt.Errorf("jarld is running; stop it first (systemctl stop jarl), then run this again")
			}
			if err != nil {
				return err
			}
			defer unlock()

			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
			fmt.Fprintf(w, "Backup:\t%s\n", src)
			describe(w, info, backupTime(src))
			if len(keyFiles) > 0 {
				fmt.Fprintf(w, "  DNSSEC keys\t%s (go back to %s)\n", plural(len(keyFiles), "file"), keysDir)
			}
			if cur, err := backup.Inspect(ctx, cfg.Database); err == nil {
				fmt.Fprintf(w, "\nCurrent database:\t%s\n", cfg.Database)
				describe(w, cur, time.Time{})
			} else if !errors.Is(err, os.ErrNotExist) {
				fmt.Fprintf(w, "\nCurrent database:\t%s (unreadable: %v)\n", cfg.Database, err)
			}
			w.Flush()
			if !yes {
				if !term.IsTerminal(int(os.Stdin.Fd())) {
					return fmt.Errorf("add --yes to restore without being asked")
				}
				fmt.Print("\nReplace the current database with this backup? [y/N] ")
				line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
				if a := strings.ToLower(strings.TrimSpace(line)); a != "y" && a != "yes" {
					return fmt.Errorf("nothing changed")
				}
			}

			db := cfg.Database
			defer newDirOwner(filepath.Dir(db)).fix(filepath.Dir(db))
			aside := db + ".before-restore-" + time.Now().UTC().Format("20060102T150405Z")
			// The database's WAL and shared-memory files belong to it:
			// they move with it, and the restored copy starts without.
			moved := false
			for _, suffix := range []string{"", "-wal", "-shm"} {
				if _, err := os.Lstat(db + suffix); err == nil {
					if err := os.Rename(db+suffix, aside+suffix); err != nil {
						return err
					}
					moved = true
				}
			}
			restoreErr := copyFile(src, db, 0o640)
			if restoreErr == nil {
				// The keys go back to their own files, not into the
				// live database.
				restoreErr = backup.DropFiles(ctx, db)
			}
			if restoreErr != nil {
				os.Remove(db)
				if moved {
					os.Rename(aside, db)
					os.Rename(aside+"-wal", db+"-wal")
					os.Rename(aside+"-shm", db+"-shm")
				}
				return fmt.Errorf("restoring: %w (the current database was left in place)", restoreErr)
			}
			if len(keyFiles) > 0 {
				n, err := restoreKeys(cfg, keysDir, keyFiles)
				if err != nil {
					return fmt.Errorf("the database was restored, but putting back the DNSSEC keys failed after %d files: %w", n, err)
				}
				fmt.Printf("Put back %s in %s.\n", plural(n, "DNSSEC key file"), keysDir)
			}
			// Open it as jarld will: an older backup gets the current
			// schema now, and the restore goes into its audit log.
			a, err := openApp(true)
			if err != nil {
				return fmt.Errorf("the backup was copied to %s but doesn't open: %w", db, err)
			}
			p := &auth.Principal{Method: "local"}
			if lp, err := a.localPrincipal(ctx, ""); err == nil {
				p = lp
			}
			a.eng.Audit(ctx, p, "system.restore", "", 0, "", "success", "database restored from "+src)
			a.close()

			fmt.Printf("\nRestored %s from %s.\n", db, src)
			if moved {
				fmt.Printf("The previous database is in %s.\n", aside)
			}
			fmt.Print(`
Next:
  systemctl start jarl
  jarl resync --dry-run     # what differs between the restored database and named's files
  jarl resync --force       # write them and reload named

Zones whose data goes back to an older state get a serial above the one
named has, so named and the secondaries load them (see "Restoring" in
docs/operations.md).
`)
			return nil
		},
	}
	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "don't ask for confirmation")
	cmd.Flags().BoolVar(&withoutKeys, "without-keys", false, "restore only the database, not the DNSSEC keys")
	return cmd
}

// runningAs reports whether this process runs as the named user.
func runningAs(name string) bool {
	u, err := user.Lookup(name)
	return err == nil && u.Uid == strconv.Itoa(os.Geteuid())
}

// restoreKeys writes DNSSEC key files from a backup into keysDir, owned
// by named's user and Jarl's file group with their backed-up modes, so
// named can use them. Directories get the owner of the zone directory
// (jarld's user), like the ones jarld makes. Files already there are
// replaced; others are left alone.
func restoreKeys(cfg *config.Config, keysDir string, files []backup.StoredFile) (int, error) {
	uid, gid := -1, -1
	if u, err := user.Lookup(cfg.BIND.NamedUser); err == nil {
		uid, _ = strconv.Atoi(u.Uid)
	} else {
		return 0, fmt.Errorf("named's user %q (bind.named_user): %w", cfg.BIND.NamedUser, err)
	}
	if g, err := user.LookupGroup(cfg.BIND.FileGroup); err == nil {
		gid, _ = strconv.Atoi(g.Gid)
	}
	dirUID, dirGID := -1, gid
	if fi, err := os.Stat(cfg.BIND.ZoneDir); err == nil {
		if st, ok := fi.Sys().(*syscall.Stat_t); ok {
			dirUID = int(st.Uid)
		}
	}
	var mkdir func(dir string) error
	mkdir = func(dir string) error {
		if _, err := os.Stat(dir); err == nil {
			return nil
		}
		if err := mkdir(filepath.Dir(dir)); err != nil {
			return err
		}
		if err := os.Mkdir(dir, 0o770); err != nil && !os.IsExist(err) {
			return err
		}
		os.Chown(dir, dirUID, dirGID)
		return os.Chmod(dir, 0o770|os.ModeSetgid)
	}
	n := 0
	for _, f := range files {
		if filepath.IsAbs(f.Path) || strings.HasPrefix(filepath.Clean(f.Path), "..") {
			return n, fmt.Errorf("refusing the file name %q in the backup", f.Path)
		}
		dst := filepath.Join(keysDir, f.Path)
		if err := mkdir(filepath.Dir(dst)); err != nil {
			return n, err
		}
		tmp := dst + ".restore-tmp"
		if err := os.WriteFile(tmp, f.Data, 0o600); err != nil {
			return n, err
		}
		if err := os.Chown(tmp, uid, gid); err != nil {
			os.Remove(tmp)
			return n, err
		}
		if err := os.Chmod(tmp, f.Mode); err != nil {
			os.Remove(tmp)
			return n, err
		}
		if err := os.Rename(tmp, dst); err != nil {
			os.Remove(tmp)
			return n, err
		}
		n++
	}
	return n, nil
}

func listBackups(dir string) error {
	files, err := backup.List(dir)
	if err != nil {
		return err
	}
	if len(files) == 0 {
		fmt.Printf("No backups in %s.\n", dir)
		return nil
	}
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
	fmt.Fprintln(w, "MADE\tAGE\tKIND\tSIZE\tFILE")
	for _, f := range files {
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", f.Time.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(f.Time)), f.Kind, size(f.Size), f.Path)
	}
	w.Flush()
	fmt.Println("\nRestore one with: jarld restore FILE   (or: jarld restore latest)")
	return nil
}

func describe(w io.Writer, in *backup.Info, made time.Time) {
	if !made.IsZero() {
		fmt.Fprintf(w, "  made\t%s (%s ago)\n", made.Local().Format("2006-01-02 15:04:05"), fmtDuration(time.Since(made)))
	}
	fmt.Fprintf(w, "  contents\t%s, %s, %s (schema %d)\n", plural(in.Zones, "zone"), plural(in.Records, "record"), plural(in.Users, "user"), in.Schema)
	if in.SignedZones > 0 {
		fmt.Fprintf(w, "  signed zones\t%d\n", in.SignedZones)
	}
	if in.LastChange != nil {
		fmt.Fprintf(w, "  last change\t%s\n", in.LastChange.Local().Format("2006-01-02 15:04:05"))
	}
}

// backupTime is when a backup was made: from its name if it has a
// backup's name, otherwise its modification time.
func backupTime(path string) time.Time {
	for _, f := range mustList(filepath.Dir(path)) {
		if f.Path == filepath.Clean(path) {
			return f.Time
		}
	}
	if fi, err := os.Stat(path); err == nil {
		return fi.ModTime()
	}
	return time.Time{}
}

func mustList(dir string) []backup.File {
	files, _ := backup.List(dir)
	return files
}

// copyFile copies src to dst through a temporary file, so dst is either
// complete or absent.
func copyFile(src, dst string, mode os.FileMode) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	tmp := dst + ".restore-tmp"
	out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
	if err != nil {
		return err
	}
	if _, err := io.Copy(out, in); err != nil {
		out.Close()
		os.Remove(tmp)
		return err
	}
	if err := out.Sync(); err != nil {
		out.Close()
		os.Remove(tmp)
		return err
	}
	if err := out.Close(); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, dst)
}

func size(n int64) string {
	switch {
	case n >= 1<<20:
		return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
	case n >= 1<<10:
		return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
	}
	return fmt.Sprintf("%d bytes", n)
}

func fmtDuration(d time.Duration) string {
	switch {
	case d >= 48*time.Hour:
		return fmt.Sprintf("%d days", int(d/(24*time.Hour)))
	case d >= 2*time.Hour:
		return fmt.Sprintf("%d hours", int(d/time.Hour))
	case d >= 2*time.Minute:
		return fmt.Sprintf("%d minutes", int(d/time.Minute))
	}
	return "under 2 minutes"
}

func plural(n int, word string) string {
	if n == 1 {
		return "1 " + word
	}
	return fmt.Sprintf("%d %ss", n, word)
}