package server import ( "context" "encoding/json" "fmt" "sync" "time" "housecarl/internal/api" "housecarl/internal/auth" "housecarl/internal/engine" "housecarl/internal/model" ) // A change made over the network is applied on probation: unless it is // confirmed in time (POST /confirm, hc confirm, the web UI's banner), it is // undone, so a change that cuts the user off from the firewall undoes // itself. Changes through the local socket (root on the firewall) need no // confirmation, but join a probation already running, so a rollback undoes // everything since the last confirmed state. The pending entries are kept // in the database: housecarld starting with any rolls them back. // Housecarl adds no rule of its own to keep the API reachable. const probationKey = "probation" type probation struct { mu sync.Mutex pending []int64 deadline time.Time timer *time.Timer // restores are nft scripts that bring back tables a takeover deleted. restores []string } // system is the principal of rollbacks. var system = &auth.Principal{User: &model.User{Username: "housecarl"}, Method: "system"} type savedProbation struct { Pending []int64 `json:"pending"` Deadline time.Time `json:"deadline"` Restores []string `json:"restores,omitempty"` } func (s *Server) saveProbation(ctx context.Context) { b, _ := json.Marshal(savedProbation{Pending: s.probation.pending, Deadline: s.probation.deadline, Restores: s.probation.restores}) if err := s.store.SetMeta(ctx, probationKey, string(b)); err != nil { s.log.Error("saving the probation failed", "err", err) } } // afterChange is called after every applied change. func (s *Server) afterChange(ctx context.Context, p *auth.Principal, res *engine.Result) { if res == nil || !res.Applied || res.HistoryID == 0 { return } s.startProbation(ctx, p, res.HistoryID, "") } // startProbation puts a change (a history entry, or a takeover's restore // script) on probation, or, through the socket with nothing pending, // makes it permanent at once. func (s *Server) startProbation(ctx context.Context, p *auth.Principal, historyID int64, restore string) { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() if p.Method == "socket" && len(pr.pending) == 0 && len(pr.restores) == 0 { s.saveBoot() return } if historyID != 0 { pr.pending = append(pr.pending, historyID) } if restore != "" { pr.restores = append(pr.restores, restore) } if p.Method != "socket" || pr.deadline.IsZero() { d := time.Duration(s.eng.State().Settings.Probation()) * time.Second if s.probationFor > 0 { d = s.probationFor } pr.deadline = s.nowFn().Add(d) if pr.timer != nil { pr.timer.Stop() } pr.timer = time.AfterFunc(d, func() { s.rollback("not confirmed in time") }) } s.saveProbation(ctx) } func (s *Server) saveBoot() { if err := s.fw.SaveBoot(s.eng.State()); err != nil { s.log.Error("saving the ruleset for the next boot failed", "err", err) } } // probationStatus describes what is pending. func (s *Server) probationStatus() api.Probation { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() out := api.Probation{Pending: append([]int64{}, pr.pending...), Takeovers: len(pr.restores)} if len(pr.pending) > 0 || len(pr.restores) > 0 { out.Deadline = pr.deadline out.SecondsLeft = int(pr.deadline.Sub(s.nowFn()).Seconds() + 0.5) if out.SecondsLeft < 0 { out.SecondsLeft = 0 } } return out } // confirm makes the pending changes permanent. func (s *Server) confirm(ctx context.Context, p *auth.Principal) (int, error) { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() n := len(pr.pending) + len(pr.restores) if n == 0 { return 0, nil } if pr.timer != nil { pr.timer.Stop() } pr.pending, pr.restores, pr.deadline, pr.timer = nil, nil, time.Time{}, nil s.saveProbation(ctx) s.saveBoot() s.eng.Audit(ctx, p, "config.confirm", "", "success", fmt.Sprintf("%d changes confirmed", n)) return n, nil } // rollback undoes the pending changes, newest first. func (s *Server) rollback(why string) { ctx := context.Background() pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() pending, restores := pr.pending, pr.restores pr.pending, pr.restores, pr.deadline, pr.timer = nil, nil, time.Time{}, nil for i := len(restores) - 1; i >= 0; i-- { if err := s.fw.ApplyScript(ctx, restores[i]); err != nil { s.log.Error("bringing back a table a takeover deleted failed", "err", err) s.eng.Audit(ctx, system, "takeover.rollback", "", "failure", err.Error()) } else { s.eng.Audit(ctx, system, "takeover.rollback", "", "success", why) s.log.Warn("brought back a table an unconfirmed takeover deleted", "why", why) } } for i := len(pending) - 1; i >= 0; i-- { id := pending[i] if _, err := s.eng.Revert(ctx, system, id, engine.Options{Via: "rollback"}); err != nil { s.log.Error("rolling back an unconfirmed change failed", "history", id, "err", err) s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "failure", err.Error()) continue } s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "success", why) s.log.Warn("rolled back an unconfirmed change", "history", id, "why", why) } s.saveProbation(ctx) } // ResumeProbation rolls back what was pending when housecarld stopped. It // returns whether anything was rolled back. func (s *Server) ResumeProbation(ctx context.Context) bool { v, err := s.store.GetMeta(ctx, probationKey) if err != nil || v == "" { return false } var saved savedProbation if json.Unmarshal([]byte(v), &saved) != nil || len(saved.Pending) == 0 && len(saved.Restores) == 0 { return false } s.probation.mu.Lock() s.probation.pending, s.probation.restores = saved.Pending, saved.Restores s.probation.mu.Unlock() s.rollback("housecarld restarted before they were confirmed") return true }