package server import ( "context" "crypto/subtle" "encoding/json" "fmt" "net/http" "sync" "time" "housecarl/internal/api" "housecarl/internal/auth" "housecarl/internal/engine" "housecarl/internal/model" ) // A change made over the network is applied on probation: unless it is // confirmed in time (POST /confirm, hc confirm, the web UI's banner), it is // undone, so a change that cuts the user off from the firewall undoes // itself. Changes through the local socket (root on the firewall) need no // confirmation, but join a probation already running, so a rollback undoes // everything since the last confirmed state. The pending entries are kept // in the database: housecarld starting with any rolls them back. // Housecarl adds no rule of its own to keep the API reachable. const probationKey = "probation" type probation struct { mu sync.Mutex pending []int64 deadline time.Time timer *time.Timer // restores are nft scripts that bring back tables a takeover deleted. restores []string // check is the latest change's confirmation token (reachCheck): only // it can confirm, since the ruleset in force is the one it tests. check *reachCheck } // reachCheck is what a client must show to confirm its change by itself: // the token, from the same user (and API token), the same address, and a // new connection (another remote address and port than the change's). type reachCheck struct { token string userID int64 username string tokenID int64 ip string remote string } // system is the principal of rollbacks. var system = &auth.Principal{User: &model.User{Username: "housecarl"}, Method: "system"} type savedProbation struct { Pending []int64 `json:"pending"` Deadline time.Time `json:"deadline"` Restores []string `json:"restores,omitempty"` } func (s *Server) saveProbation(ctx context.Context) { b, _ := json.Marshal(savedProbation{Pending: s.probation.pending, Deadline: s.probation.deadline, Restores: s.probation.restores}) if err := s.store.SetMeta(ctx, probationKey, string(b)); err != nil { s.log.Error("saving the probation failed", "err", err) } } // afterChange is called after every applied change. func (s *Server) afterChange(ctx context.Context, p *auth.Principal, res *engine.Result) { if res == nil || !res.Applied || res.HistoryID == 0 { return } s.startProbation(ctx, p, res.HistoryID, "") } // startProbation puts a change (a history entry, or a takeover's restore // script) on probation, or, through the socket with nothing pending, // makes it permanent at once. func (s *Server) startProbation(ctx context.Context, p *auth.Principal, historyID int64, restore string) { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() if p.Method == "socket" && len(pr.pending) == 0 && len(pr.restores) == 0 { s.saveBoot() return } if historyID != 0 { pr.pending = append(pr.pending, historyID) } if restore != "" { pr.restores = append(pr.restores, restore) } if p.Method != "socket" || pr.deadline.IsZero() { d := time.Duration(s.eng.State().Settings.Probation()) * time.Second if s.probationFor > 0 { d = s.probationFor } pr.deadline = s.nowFn().Add(d) if pr.timer != nil { pr.timer.Stop() } pr.timer = time.AfterFunc(d, func() { s.rollback("not confirmed in time") }) } s.saveProbation(ctx) } // issueCheck gives the client that made a network change on probation a // token to confirm it by itself, unless the settings want a person to // confirm. The answer then closes its connection (HTTP/2: a GOAWAY), so // the confirmation has to open a new one through the new ruleset; an // established connection would get through whatever the change did. func (s *Server) issueCheck(w http.ResponseWriter, r *http.Request, p *auth.Principal, res *engine.Result) string { if p == nil || p.Method == "socket" || (res != nil && !res.Applied) || s.eng.State().Settings.ManualConfirm { return "" } pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() if len(pr.pending) == 0 && len(pr.restores) == 0 { return "" } c := &reachCheck{token: auth.RandomString(24), username: p.Name(), ip: s.clientIP(r), remote: r.RemoteAddr} if p.User != nil { c.userID = p.User.ID } if p.Token != nil { c.tokenID = p.Token.ID } pr.check = c w.Header().Set("Connection", "close") return c.token } // confirmToken confirms with a change's token: the proof that the client // that made it still reaches the API, from where it made it. func (s *Server) confirmToken(r *http.Request, p *auth.Principal, token string) (int, error) { pr := &s.probation pr.mu.Lock() c := pr.check pending := len(pr.pending) + len(pr.restores) pr.mu.Unlock() var tokenID int64 if p.Token != nil { tokenID = p.Token.ID } switch { case pending == 0: return 0, nil case c == nil || subtle.ConstantTimeCompare([]byte(c.token), []byte(token)) != 1: return 0, conflict("this confirmation is for an earlier change: a newer one is on probation, and only its own client (or a person) can confirm it") case p.User == nil || p.User.ID != c.userID || p.Name() != c.username || tokenID != c.tokenID: return 0, forbidden("only the client that made the change can confirm it this way") case s.clientIP(r) != c.ip: return 0, forbidden("the confirmation comes from %s, the change from %s: it must come from the same place", s.clientIP(r), c.ip) case r.RemoteAddr == c.remote: return 0, conflict("the confirmation came over the change's own connection, which proves nothing; it must open a new one") } return s.confirmWith(r.Context(), p, fmt.Sprintf("confirmed automatically: %s reached the API again over a new connection", c.ip)) } func (s *Server) saveBoot() { if err := s.fw.SaveBoot(s.eng.State()); err != nil { s.log.Error("saving the ruleset for the next boot failed", "err", err) } } // probationStatus describes what is pending. func (s *Server) probationStatus() api.Probation { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() out := api.Probation{Pending: append([]int64{}, pr.pending...), Takeovers: len(pr.restores)} if len(pr.pending) > 0 || len(pr.restores) > 0 { out.Deadline = pr.deadline out.SecondsLeft = int(pr.deadline.Sub(s.nowFn()).Seconds() + 0.5) if out.SecondsLeft < 0 { out.SecondsLeft = 0 } } return out } // confirm makes the pending changes permanent (a person's confirmation). func (s *Server) confirm(ctx context.Context, p *auth.Principal) (int, error) { return s.confirmWith(ctx, p, "") } func (s *Server) confirmWith(ctx context.Context, p *auth.Principal, how string) (int, error) { pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() n := len(pr.pending) + len(pr.restores) if n == 0 { return 0, nil } if pr.timer != nil { pr.timer.Stop() } pr.pending, pr.restores, pr.deadline, pr.timer, pr.check = nil, nil, time.Time{}, nil, nil s.saveProbation(ctx) s.saveBoot() detail := fmt.Sprintf("%d changes confirmed", n) if how != "" { detail += "; " + how } s.eng.Audit(ctx, p, "config.confirm", "", "success", detail) return n, nil } // rollback undoes the pending changes, newest first. func (s *Server) rollback(why string) { ctx := context.Background() pr := &s.probation pr.mu.Lock() defer pr.mu.Unlock() pending, restores := pr.pending, pr.restores pr.pending, pr.restores, pr.deadline, pr.timer, pr.check = nil, nil, time.Time{}, nil, nil for i := len(restores) - 1; i >= 0; i-- { if err := s.fw.ApplyScript(ctx, restores[i]); err != nil { s.log.Error("bringing back a table a takeover deleted failed", "err", err) s.eng.Audit(ctx, system, "takeover.rollback", "", "failure", err.Error()) } else { s.eng.Audit(ctx, system, "takeover.rollback", "", "success", why) s.log.Warn("brought back a table an unconfirmed takeover deleted", "why", why) } } for i := len(pending) - 1; i >= 0; i-- { id := pending[i] if _, err := s.eng.Revert(ctx, system, id, engine.Options{Via: "rollback"}); err != nil { s.log.Error("rolling back an unconfirmed change failed", "history", id, "err", err) s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "failure", err.Error()) continue } s.eng.Audit(ctx, system, "config.rollback", fmt.Sprintf("history %d", id), "success", why) s.log.Warn("rolled back an unconfirmed change", "history", id, "why", why) } s.saveProbation(ctx) } // ResumeProbation rolls back what was pending when housecarld stopped. It // returns whether anything was rolled back. func (s *Server) ResumeProbation(ctx context.Context) bool { v, err := s.store.GetMeta(ctx, probationKey) if err != nil || v == "" { return false } var saved savedProbation if json.Unmarshal([]byte(v), &saved) != nil || len(saved.Pending) == 0 && len(saved.Restores) == 0 { return false } s.probation.mu.Lock() s.probation.pending, s.probation.restores = saved.Pending, saved.Restores s.probation.mu.Unlock() s.rollback("housecarld restarted before they were confirmed") return true }