jarl/internal/store/store.go

// Package store persists Jarl's data in SQLite.
//
// The configuration (views, zones, records, ...) is loaded into a
// model.State as a whole and saved back by diffing two states, so the rest
// of Jarl works on plain Go values. Users, tokens, sessions, changesets and
// the audit log are accessed directly.
package store

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	_ "modernc.org/sqlite"
)

// ErrNotFound is returned when a row does not exist.
var ErrNotFound = errors.New("not found")

// Store is a Jarl database.
type Store struct {
	db *sql.DB
}

// DB exposes the underlying database (for backups and tests).
func (s *Store) DB() *sql.DB { return s.db }

// Open opens (creating if needed) the database at path and applies
// migrations. Use ":memory:" for a throwaway in-memory database.
func Open(path string) (*Store, error) {
	dsn := "file:" + path + "?_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)&_txlock=immediate"
	if path == ":memory:" {
		dsn = "file::memory:?_pragma=foreign_keys(1)&_txlock=immediate"
	} else {
		dsn += "&_pragma=journal_mode(WAL)"
	}
	db, err := sql.Open("sqlite", dsn)
	if err != nil {
		return nil, err
	}
	if path == ":memory:" {
		// Every connection would get its own in-memory database.
		db.SetMaxOpenConns(1)
	}
	s := &Store{db: db}
	if err := s.migrate(context.Background()); err != nil {
		db.Close()
		return nil, fmt.Errorf("migrating database: %w", err)
	}
	return s, nil
}

// OpenExisting opens an existing database without creating it or
// applying migrations, for commands that work alongside a running jarld
// that may be older than they are ('jarld backup').
func OpenExisting(path string) (*Store, error) {
	db, err := sql.Open("sqlite", "file:"+path+"?mode=rw&_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)&_txlock=immediate")
	if err != nil {
		return nil, err
	}
	if err := db.Ping(); err != nil {
		db.Close()
		return nil, err
	}
	return &Store{db: db}, nil
}

// SchemaVersion is the database schema version this Jarl uses.
func SchemaVersion() int { return len(migrations) }

// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }

var migrations = []string{
	// 1: initial schema
	`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE seq (name TEXT PRIMARY KEY, value INTEGER NOT NULL);
INSERT INTO seq (name, value) VALUES ('object', 0);

CREATE TABLE views (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	position INTEGER NOT NULL,
	match_clients TEXT NOT NULL DEFAULT '[]',
	match_destinations TEXT NOT NULL DEFAULT '[]',
	match_recursive_only INTEGER,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE acls (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	elements TEXT NOT NULL DEFAULT '[]',
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE tsig_keys (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	algorithm TEXT NOT NULL,
	secret TEXT NOT NULL,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE zones (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	view_id INTEGER NOT NULL DEFAULT 0,
	type TEXT NOT NULL,
	in_view TEXT NOT NULL DEFAULT '',
	file TEXT NOT NULL DEFAULT '',
	soa_mname TEXT NOT NULL DEFAULT '', soa_rname TEXT NOT NULL DEFAULT '',
	soa_serial INTEGER NOT NULL DEFAULT 0, soa_refresh INTEGER NOT NULL DEFAULT 0,
	soa_retry INTEGER NOT NULL DEFAULT 0, soa_expire INTEGER NOT NULL DEFAULT 0,
	soa_minimum INTEGER NOT NULL DEFAULT 0,
	default_ttl INTEGER NOT NULL DEFAULT 3600,
	serial_policy TEXT NOT NULL DEFAULT '',
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE INDEX zones_name ON zones(name);
CREATE TABLE records (
	id INTEGER PRIMARY KEY,
	zone_id INTEGER NOT NULL,
	name TEXT NOT NULL,
	type TEXT NOT NULL,
	ttl INTEGER,
	data TEXT NOT NULL,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE INDEX records_zone ON records(zone_id, name);
CREATE TABLE options (
	id INTEGER PRIMARY KEY,
	scope TEXT NOT NULL,
	scope_id INTEGER NOT NULL DEFAULT 0,
	name TEXT NOT NULL,
	value TEXT NOT NULL DEFAULT '',
	position INTEGER NOT NULL DEFAULT 0,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);

CREATE TABLE users (
	id INTEGER PRIMARY KEY AUTOINCREMENT,
	username TEXT NOT NULL UNIQUE COLLATE NOCASE,
	password_hash TEXT NOT NULL DEFAULT '',
	admin INTEGER NOT NULL DEFAULT 0,
	grants TEXT NOT NULL DEFAULT '[]',
	disabled INTEGER NOT NULL DEFAULT 0,
	comment TEXT NOT NULL DEFAULT '',
	created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	last_login_at TEXT
);
CREATE TABLE tokens (
	id INTEGER PRIMARY KEY AUTOINCREMENT,
	user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
	name TEXT NOT NULL,
	public_id TEXT NOT NULL UNIQUE,
	secret_hash TEXT NOT NULL,
	grants TEXT,
	expires_at TEXT,
	created_at TEXT NOT NULL,
	last_used_at TEXT,
	last_used_ip TEXT NOT NULL DEFAULT '',
	revoked_at TEXT
);
CREATE TABLE sessions (
	id_hash TEXT PRIMARY KEY,
	user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
	csrf_token TEXT NOT NULL,
	created_at TEXT NOT NULL,
	expires_at TEXT NOT NULL,
	last_seen_at TEXT NOT NULL,
	ip TEXT NOT NULL DEFAULT '',
	user_agent TEXT NOT NULL DEFAULT ''
);
CREATE TABLE changesets (
	id INTEGER PRIMARY KEY AUTOINCREMENT,
	description TEXT NOT NULL DEFAULT '',
	status TEXT NOT NULL,
	user_id INTEGER NOT NULL DEFAULT 0,
	username TEXT NOT NULL DEFAULT '',
	via TEXT NOT NULL DEFAULT '',
	ops TEXT NOT NULL DEFAULT '[]',
	created_at TEXT NOT NULL,
	updated_at TEXT NOT NULL,
	applied_at TEXT,
	applied_by TEXT NOT NULL DEFAULT '',
	result TEXT,
	diff TEXT NOT NULL DEFAULT ''
);
CREATE INDEX changesets_status ON changesets(status, user_id);
CREATE TABLE audit (
	id INTEGER PRIMARY KEY AUTOINCREMENT,
	time TEXT NOT NULL,
	user_id INTEGER NOT NULL DEFAULT 0,
	username TEXT NOT NULL DEFAULT '',
	auth TEXT NOT NULL DEFAULT '',
	token_id INTEGER NOT NULL DEFAULT 0,
	token_name TEXT NOT NULL DEFAULT '',
	ip TEXT NOT NULL DEFAULT '',
	user_agent TEXT NOT NULL DEFAULT '',
	action TEXT NOT NULL,
	object_type TEXT NOT NULL DEFAULT '',
	object_id INTEGER NOT NULL DEFAULT 0,
	object_name TEXT NOT NULL DEFAULT '',
	changeset_id INTEGER NOT NULL DEFAULT 0,
	outcome TEXT NOT NULL,
	message TEXT NOT NULL DEFAULT '',
	details TEXT
);
CREATE INDEX audit_time ON audit(time);
CREATE INDEX audit_user ON audit(username, time);
CREATE TABLE deployed_files (
	path TEXT PRIMARY KEY,
	sha256 TEXT NOT NULL,
	size INTEGER NOT NULL,
	kind TEXT NOT NULL,
	written_at TEXT NOT NULL,
	changeset_id INTEGER NOT NULL DEFAULT 0
);
`,
	// 2: token rotation. A rotated token gets a new secret; the previous
	// one may stay valid for a grace period.
	`
ALTER TABLE tokens ADD COLUMN rotated_at TEXT;
ALTER TABLE tokens ADD COLUMN prev_secret_hash TEXT NOT NULL DEFAULT '';
ALTER TABLE tokens ADD COLUMN prev_expires_at TEXT;
`,
	// 3: DNSSEC signing (the zone's dnssec-policy).
	`
ALTER TABLE zones ADD COLUMN dnssec_policy TEXT NOT NULL DEFAULT '';
`,
	// 4: two-factor login with TOTP. totp_pending holds a secret being set
	// up until a code confirms it; totp_last_step is the newest time step
	// used, so a code works only once. sessions.totp_at is when the
	// session last proved a code (for actions that ask for a recent one).
	`
ALTER TABLE users ADD COLUMN totp_secret TEXT NOT NULL DEFAULT '';
ALTER TABLE users ADD COLUMN totp_pending TEXT NOT NULL DEFAULT '';
ALTER TABLE users ADD COLUMN totp_enabled_at TEXT;
ALTER TABLE users ADD COLUMN totp_last_step INTEGER NOT NULL DEFAULT 0;
ALTER TABLE sessions ADD COLUMN totp_at TEXT;
CREATE TABLE recovery_codes (
	user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
	code_hash TEXT NOT NULL,
	used_at TEXT,
	PRIMARY KEY (user_id, code_hash)
);
`,
	// 5: disabled records, kept but left out of the zone.
	`
ALTER TABLE records ADD COLUMN disabled INTEGER NOT NULL DEFAULT 0;
`,
	// 6: blocking (docs/blocking.md). blocklist_status holds what jarld
	// learns downloading lists; it is not configuration.
	`
CREATE TABLE blocklists (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	url TEXT NOT NULL,
	view_ids TEXT NOT NULL DEFAULT '[]',
	disabled INTEGER NOT NULL DEFAULT 0,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE block_rules (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	action TEXT NOT NULL,
	exact INTEGER NOT NULL DEFAULT 0,
	view_ids TEXT NOT NULL DEFAULT '[]',
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE blocklist_status (
	id INTEGER PRIMARY KEY,
	url TEXT NOT NULL DEFAULT '',
	etag TEXT NOT NULL DEFAULT '',
	last_modified TEXT NOT NULL DEFAULT '',
	checked_at TEXT,
	updated_at TEXT,
	entries INTEGER NOT NULL DEFAULT 0,
	records INTEGER NOT NULL DEFAULT 0,
	skipped INTEGER NOT NULL DEFAULT 0,
	sha256 TEXT NOT NULL DEFAULT '',
	error TEXT NOT NULL DEFAULT ''
);
`,
	// 7: disabled zones, kept but not loaded by named.
	`
ALTER TABLE zones ADD COLUMN disabled INTEGER NOT NULL DEFAULT 0;
`,
	// 8: blocking steps and client groups (docs/blocking.md).
	`
ALTER TABLE blocklists ADD COLUMN step_id INTEGER NOT NULL DEFAULT 0;
CREATE TABLE block_steps (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	position INTEGER NOT NULL DEFAULT 0,
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
CREATE TABLE client_groups (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL,
	members TEXT NOT NULL DEFAULT '[]',
	step_id INTEGER NOT NULL DEFAULT 0,
	view_ids TEXT NOT NULL DEFAULT '[]',
	comment TEXT NOT NULL DEFAULT '',
	version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
`,
	// 9: internet exposure (docs/exposure.md): a view's override and its
	// acknowledged findings.
	`
ALTER TABLE views ADD COLUMN exposure TEXT NOT NULL DEFAULT '';
ALTER TABLE views ADD COLUMN acknowledged TEXT NOT NULL DEFAULT '[]';
`,
	// 10: TLS certificates (docs/certificates.md). cert_pem and key_pem
	// hold the current certificate of one Jarl obtains.
	`
CREATE TABLE certificates (
	id INTEGER PRIMARY KEY,
	name TEXT NOT NULL UNIQUE,
	source TEXT NOT NULL,
	domains TEXT NOT NULL DEFAULT '[]',
	cert_file TEXT NOT NULL DEFAULT '',
	key_file TEXT NOT NULL DEFAULT '',
	uses TEXT NOT NULL DEFAULT '[]',
	challenge TEXT NOT NULL DEFAULT '',
	key_type TEXT NOT NULL DEFAULT '',
	comment TEXT NOT NULL DEFAULT '',
	cert_pem TEXT NOT NULL DEFAULT '',
	key_pem TEXT NOT NULL DEFAULT '',
	names TEXT NOT NULL DEFAULT '[]',
	not_before TEXT, not_after TEXT,
	issuer TEXT NOT NULL DEFAULT '',
	serial TEXT NOT NULL DEFAULT '',
	renew_at TEXT, ari_checked_at TEXT,
	last_attempt TEXT, last_success TEXT,
	last_error TEXT NOT NULL DEFAULT '',
	failures INTEGER NOT NULL DEFAULT 0,
	last_routes TEXT NOT NULL DEFAULT '[]',
	created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
	created_by TEXT NOT NULL DEFAULT '', updated_by TEXT NOT NULL DEFAULT ''
);
`,
	// 11: linked zones (docs/views.md#linked-zones).
	`
ALTER TABLE zones ADD COLUMN link_group INTEGER NOT NULL DEFAULT 0;
ALTER TABLE records ADD COLUMN shared_id INTEGER NOT NULL DEFAULT 0;
`,
	// 12: PTR records kept in step per zone (docs/reverse.md).
	`
ALTER TABLE zones ADD COLUMN auto_ptr INTEGER NOT NULL DEFAULT 0;
`,
}

func (s *Store) migrate(ctx context.Context) error {
	if _, err := s.db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`); err != nil {
		return err
	}
	var v int
	err := s.db.QueryRowContext(ctx, `SELECT version FROM schema_version`).Scan(&v)
	if errors.Is(err, sql.ErrNoRows) {
		if _, err := s.db.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (0)`); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	if v > len(migrations) {
		return fmt.Errorf("database schema version %d is newer than this Jarl (%d); upgrade Jarl", v, len(migrations))
	}
	for i := v; i < len(migrations); i++ {
		tx, err := s.db.BeginTx(ctx, nil)
		if err != nil {
			return err
		}
		if _, err := tx.ExecContext(ctx, migrations[i]); err != nil {
			tx.Rollback()
			return fmt.Errorf("migration %d: %w", i+1, err)
		}
		if _, err := tx.ExecContext(ctx, `UPDATE schema_version SET version = ?`, i+1); err != nil {
			tx.Rollback()
			return err
		}
		if err := tx.Commit(); err != nil {
			return err
		}
	}
	return nil
}

// Tx is a database transaction.
type Tx struct {
	tx  *sql.Tx
	ctx context.Context
}

// InTx runs fn in a transaction, committing if it returns nil.
func (s *Store) InTx(ctx context.Context, fn func(tx *Tx) error) error {
	t, err := s.db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	if err := fn(&Tx{tx: t, ctx: ctx}); err != nil {
		t.Rollback()
		return err
	}
	return t.Commit()
}

type execer interface {
	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}

// AllocateIDs reserves n object IDs.
func (s *Store) AllocateIDs(ctx context.Context, n int) ([]int64, error) {
	if n <= 0 {
		return nil, nil
	}
	var last int64
	err := s.db.QueryRowContext(ctx, `UPDATE seq SET value = value + ? WHERE name = 'object' RETURNING value`, n).Scan(&last)
	if err != nil {
		return nil, err
	}
	ids := make([]int64, n)
	for i := range ids {
		ids[i] = last - int64(n) + 1 + int64(i)
	}
	return ids, nil
}

// GetMeta reads a meta value.
func (s *Store) GetMeta(ctx context.Context, key string) (string, error) {
	var v string
	err := s.db.QueryRowContext(ctx, `SELECT value FROM meta WHERE key = ?`, key).Scan(&v)
	if errors.Is(err, sql.ErrNoRows) {
		return "", ErrNotFound
	}
	return v, err
}

// SetMeta writes a meta value.
func (s *Store) SetMeta(ctx context.Context, key, value string) error {
	return setMeta(ctx, s.db, key, value)
}

func setMeta(ctx context.Context, e execer, key, value string) error {
	_, err := e.ExecContext(ctx, `INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value)
	return err
}

// Backup writes a consistent copy of the database to path.
func (s *Store) Backup(ctx context.Context, path string) error {
	_, err := s.db.ExecContext(ctx, `VACUUM INTO ?`, path)
	return err
}

// time helpers

func ts(t time.Time) string { return t.UTC().Format(time.RFC3339Nano) }

func tsPtr(t *time.Time) any {
	if t == nil {
		return nil
	}
	return ts(*t)
}

func parseTS(s string) time.Time {
	t, _ := time.Parse(time.RFC3339Nano, s)
	return t
}

func parseTSPtr(s sql.NullString) *time.Time {
	if !s.Valid || s.String == "" {
		return nil
	}
	t := parseTS(s.String)
	return &t
}

func jsonText(v any) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic(err)
	}
	return string(b)
}

func boolInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

func placeholders(n int) string {
	return strings.TrimSuffix(strings.Repeat("?,", n), ",")
}

// BackupRecord is a backup that 'jarld backup' made (or failed to make),
// kept in the meta table for the dashboard.
type BackupRecord struct {
	At    time.Time `json:"at"`
	File  string    `json:"file,omitempty"`
	Size  int64     `json:"size,omitempty"`
	Error string    `json:"error,omitempty"`
}

const (
	metaBackupOK     = "backup.last_ok"
	metaBackupFailed = "backup.last_failed"
)

// RecordBackup stores the outcome of a backup: r.Error empty for a
// successful one.
func (s *Store) RecordBackup(ctx context.Context, r BackupRecord) error {
	key := metaBackupOK
	if r.Error != "" {
		key = metaBackupFailed
	}
	b, err := json.Marshal(r)
	if err != nil {
		return err
	}
	return s.SetMeta(ctx, key, string(b))
}

// LastBackups returns the newest successful and failed backups (nil for
// none).
func (s *Store) LastBackups(ctx context.Context) (ok, failed *BackupRecord) {
	get := func(key string) *BackupRecord {
		v, err := s.GetMeta(ctx, key)
		if err != nil {
			return nil
		}
		var r BackupRecord
		if json.Unmarshal([]byte(v), &r) != nil {
			return nil
		}
		return &r
	}
	return get(metaBackupOK), get(metaBackupFailed)
}