housecarl/internal/server/probation.go

package server

import (
	"context"
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"

	"housecarl/internal/api"
	"housecarl/internal/auth"
	"housecarl/internal/engine"
	"housecarl/internal/model"
)

// A change made over the network is applied on probation: unless it is
// confirmed in time (POST /confirm, hc confirm, the web UI's banner), it is
// undone, so a change that cuts the user off from the firewall undoes
// itself. Changes through the local socket (root on the firewall) need no
// confirmation, but join a probation already running, so a rollback undoes
// everything since the last confirmed state. The pending entries are kept
// in the database: housecarld starting with any rolls them back.
// Housecarl adds no rule of its own to keep the API reachable.

const probationKey = "probation"

type probation struct {
	mu       sync.Mutex
	pending  []int64
	deadline time.Time
	timer    *time.Timer
	// restores are nft scripts that bring back tables a takeover deleted.
	restores []string
	// check is the latest change's confirmation token (reachCheck): only
	// it can confirm, since the ruleset in force is the one it tests.
	check *reachCheck
}

// reachCheck is what a client must show to confirm its change by itself:
// the token, from the same user (and API token), the same address, and a
// new connection (another remote address and port than the change's).
type reachCheck struct {
	token    string
	userID   int64
	username string
	tokenID  int64
	ip       string
	remote   string
}

// system is the principal of rollbacks.
var system = &auth.Principal{User: &model.User{Username: "housecarl"}, Method: "system"}

type savedProbation struct {
	Pending  []int64   `json:"pending"`
	Deadline time.Time `json:"deadline"`
	Restores []string  `json:"restores,omitempty"`
}

func (s *Server) saveProbation(ctx context.Context) {
	b, _ := json.Marshal(savedProbation{Pending: s.probation.pending, Deadline: s.probation.deadline, Restores: s.probation.restores})
	if err := s.store.SetMeta(ctx, probationKey, string(b)); err != nil {
		s.log.Error("saving the probation failed", "err", err)
	}
}

// afterChange is called after every applied change.
func (s *Server) afterChange(ctx context.Context, p *auth.Principal, res *engine.Result) {
	if res == nil || !res.Applied || res.HistoryID == 0 {
		return
	}
	s.startProbation(ctx, p, res.HistoryID, "")
}

// startProbation puts a change (a history entry, or a takeover's restore
// script) on probation, or, through the socket with nothing pending,
// makes it permanent at once.
func (s *Server) startProbation(ctx context.Context, p *auth.Principal, historyID int64, restore string) {
	pr := &s.probation
	pr.mu.Lock()
	defer pr.mu.Unlock()
	// In shadow mode a change can't cut anyone off: it blocks nothing.
	if s.cfg.Shadow || p.Method == "socket" && len(pr.pending) == 0 && len(pr.restores) == 0 {
		s.saveBoot()
		return
	}
	if historyID != 0 {
		pr.pending = append(pr.pending, historyID)
	}
	if restore != "" {
		pr.restores = append(pr.restores, restore)
	}
	if p.Method != "socket" || pr.deadline.IsZero() {
		d := time.Duration(s.eng.State().Settings.Probation()) * time.Second
		if s.probationFor > 0 {
			d = s.probationFor
		}
		pr.deadline = s.nowFn().Add(d)
		if pr.timer != nil {
			pr.timer.Stop()
		}
		pr.timer = time.AfterFunc(d, func() { s.rollback("not confirmed in time") })
	}
	s.saveProbation(ctx)
}

// issueCheck gives the client that made a network change on probation a
// token to confirm it by itself, unless the settings want a person to
// confirm. The answer then closes its connection (HTTP/2: a GOAWAY), so
// the confirmation has to open a new one through the new ruleset; an
// established connection would get through whatever the change did.
func (s *Server) issueCheck(w http.ResponseWriter, r *http.Request, p *auth.Principal, res *engine.Result) string {
	if p == nil || p.Method == "socket" || (res != nil && !res.Applied) || s.eng.State().Settings.ManualConfirm {
		return ""
	}
	pr := &s.probation
	pr.mu.Lock()
	defer pr.mu.Unlock()
	if len(pr.pending) == 0 && len(pr.restores) == 0 {
		return ""
	}
	c := &reachCheck{token: auth.RandomString(24), username: p.Name(), ip: s.clientIP(r), remote: r.RemoteAddr}
	if p.User != nil {
		c.userID = p.User.ID
	}
	if p.Token != nil {
		c.tokenID = p.Token.ID
	}
	pr.check = c
	w.Header().Set("Connection", "close")
	return c.token
}

// confirmToken confirms with a change's token: the proof that the client
// that made it still reaches the API, from where it made it.
func (s *Server) confirmToken(r *http.Request, p *auth.Principal, token string) (int, error) {
	pr := &s.probation
	pr.mu.Lock()
	c := pr.check
	pending := len(pr.pending) + len(pr.restores)
	pr.mu.Unlock()
	var tokenID int64
	if p.Token != nil {
		tokenID = p.Token.ID
	}
	switch {
	case pending == 0:
		return 0, nil
	case c == nil || subtle.ConstantTimeCompare([]byte(c.token), []byte(token)) != 1:
		return 0, conflict("this confirmation is for an earlier change: a newer one is on probation, and only its own client (or a person) can confirm it")
	case p.User == nil || p.User.ID != c.userID || p.Name() != c.username || tokenID != c.tokenID:
		return 0, forbidden("only the client that made the change can confirm it this way")
	case s.clientIP(r) != c.ip:
		return 0, forbidden("the confirmation comes from %s, the change from %s: it must come from the same place", s.clientIP(r), c.ip)
	case r.RemoteAddr == c.remote:
		return 0, conflict("the confirmation came over the change's own connection, which proves nothing; it must open a new one")
	}
	return s.confirmWith(r.Context(), p, fmt.Sprintf("confirmed automatically: %s reached the API again over a new connection", c.ip))
}

func (s *Server) saveBoot() {
	if err := s.fw.SaveBoot(s.eng.State()); err != nil {
		s.log.Error("saving the ruleset for the next boot failed", "err", err)
	}
}

// probationStatus describes what is pending.
func (s *Server) probationStatus() api.Probation {
	pr := &s.probation
	pr.mu.Lock()
	defer pr.mu.Unlock()
	out := api.Probation{Pending: append([]int64{}, pr.pending...), Takeovers: len(pr.restores)}
	if len(pr.pending) > 0 || len(pr.restores) > 0 {
		out.Deadline = pr.deadline
		out.SecondsLeft = int(pr.deadline.Sub(s.nowFn()).Seconds() + 0.5)
		if out.SecondsLeft < 0 {
			out.SecondsLeft = 0
		}
	}
	return out
}

// confirm makes the pending changes permanent (a person's confirmation).
func (s *Server) confirm(ctx context.Context, p *auth.Principal) (int, error) {
	return s.confirmWith(ctx, p, "")
}

func (s *Server) confirmWith(ctx context.Context, p *auth.Principal, how string) (int, error) {
	pr := &s.probation
	pr.mu.Lock()
	defer pr.mu.Unlock()
	n := len(pr.pending) + len(pr.restores)
	if n == 0 {
		return 0, nil
	}
	if pr.timer != nil {
		pr.timer.Stop()
	}
	pr.pending, pr.restores, pr.deadline, pr.timer, pr.check = nil, nil, time.Time{}, nil, nil
	s.saveProbation(ctx)
	s.saveBoot()
	detail := fmt.Sprintf("%d changes confirmed", n)
	if how != "" {
		detail += "; " + how
	}
	s.eng.Audit(ctx, p, "config.confirm", "", "success", detail)
	return n, nil
}

// rollback undoes the pending changes, newest first.
func (s *Server) rollback(why string) {
	ctx := context.Background()
	pr := &s.probation
	pr.mu.Lock()
	defer pr.mu.Unlock()
	pending, restores := pr.pending, pr.restores
	pr.pending, pr.restores, pr.deadline, pr.timer, pr.check = nil, nil, time.Time{}, nil, nil
	for i := len(restores) - 1; i >= 0; i-- {
		if err := s.fw.ApplyScript(ctx, restores[i]); err != nil {
			s.log.Error("bringing back a table a takeover deleted failed", "err", err)
			s.eng.Audit(ctx, system, "takeover.rollback", "", "failure", err.Error())
		} else {
			s.eng.Audit(ctx, system, "takeover.rollback", "", "success", why)
			s.log.Warn("brought back a table an unconfirmed takeover deleted", "why", why)
		}
	}
	for i := len(pending) - 1; i >= 0; i-- {
		id := pending[i]
		if _, err := s.eng.Revert(ctx, system, id, engine.Options{Via: "rollback"}); err != nil {
			s.log.Error("rolling back an unconfirmed change failed", "history", id, "err", err)
			s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "failure", err.Error())
			continue
		}
		s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "success", why)
		s.log.Warn("rolled back an unconfirmed change", "history", id, "why", why)
	}
	s.saveProbation(ctx)
}

// ResumeProbation rolls back what was pending when housecarld stopped. It
// returns whether anything was rolled back.
func (s *Server) ResumeProbation(ctx context.Context) bool {
	v, err := s.store.GetMeta(ctx, probationKey)
	if err != nil || v == "" {
		return false
	}
	var saved savedProbation
	if json.Unmarshal([]byte(v), &saved) != nil || len(saved.Pending) == 0 && len(saved.Restores) == 0 {
		return false
	}
	s.probation.mu.Lock()
	s.probation.pending, s.probation.restores = saved.Pending, saved.Restores
	s.probation.mu.Unlock()
	s.rollback("housecarld restarted before they were confirmed")
	return true
}