From ecf4129877eca72a3763f99dbcd81fee3e89a38c Mon Sep 17 00:00:00 2001 From: abrown38 Date: Sun, 25 Oct 2026 01:32:20 +0000 Subject: [PATCH] Reload guard.yaml without losing the bounties systemctl reload guard, guard reload or SIGHUP reads the configuration again: a watch keeps its bounties and counts under the same name and takes its new settings at once; its source starts again only if it changed; watches gone stop, new ones start; a file with a mistake is refused. The Housecarl connection, state, socket and log wait for a restart, and guard reload says so. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +- README.md | 2 + cmd/guard/main.go | 103 +++++++++++++--- deploy/systemd/guard.service | 2 + docs/cli.md | 1 + docs/configuration.md | 28 ++++- internal/bounty/bounty.go | 7 ++ internal/config/example.yaml | 6 +- internal/daemon/daemon.go | 219 ++++++++++++++++++++++----------- internal/daemon/daemon_test.go | 137 +++++++++++++++++++++ internal/daemon/reload.go | 185 ++++++++++++++++++++++++++++ 11 files changed, 607 insertions(+), 93 deletions(-) create mode 100644 internal/daemon/reload.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed4970..5554392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,14 @@ All notable changes to Guard are documented here. The format follows `postfix-sasl`, `dovecot`, `nginx-auth`, `housecarl-drops` (Housecarl's log of dropped packets, TCP only by default). - Commands: `guard run`, `check`, `config [--example]`, `presets`, - `test` (a watch's patterns over a log), `status` (through a local - socket), `bans`, `ban`, `unban`, `version`. + `test` (a watch's patterns over a log), `status` and `reload` (through + a local socket), `bans`, `ban`, `unban`, `version`. +- Reloading: `systemctl reload guard`, `guard reload` or SIGHUP reads + guard.yaml again without losing the bounties. A watch keeps its + bounties under the same name and takes its new settings at once; its + source starts again only if it changed; watches gone stop, new ones + start. A file with a mistake is refused. The Housecarl connection, + state, socket and log wait for a restart (`guard reload` says so). - A Debian package with a sandboxed systemd unit, as the `guard` user in the `adm` and `systemd-journal` groups, published with each release to the Gitea Debian registry (`stable`, or `testing` for a release diff --git a/README.md b/README.md index 4f58665..989e84a 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ watch: * **IPv6 by /64**, as the ban list says: one customer usually has a whole /64 and moves around in it. * **Housecarl away?** Bans wait and are sent once it answers. +* **`systemctl reload guard`** puts a changed guard.yaml to work + without losing what Guard has counted. * **`guard test`** shows what a watch's patterns find in a log, and whom they would ban, before you trust it. diff --git a/cmd/guard/main.go b/cmd/guard/main.go index a662c72..eb65c21 100644 --- a/cmd/guard/main.go +++ b/cmd/guard/main.go @@ -53,6 +53,7 @@ keeps it from this machine (or from what the list says). guard check check guard.yaml and reach Housecarl guard test ssh /var/log/auth.log what a watch's patterns find in a log guard status the watches, their highest bounties, recent bans + guard reload read guard.yaml again, keeping the bounties Exit codes: 0 ok, 1 error.`, SilenceUsage: true, @@ -61,7 +62,7 @@ Exit codes: 0 ok, 1 error.`, } root.PersistentFlags().StringVarP(&flagConfig, "config", "c", config.DefaultPath, "configuration file") root.PersistentFlags().BoolVar(&flagJSON, "json", false, "print JSON instead of tables") - root.AddCommand(runCmd(), checkCmd(), configCmd(), presetsCmd(), testCmd(), statusCmd(), bansCmd(), banCmd(), unbanCmd(), versionCmd()) + root.AddCommand(runCmd(), checkCmd(), configCmd(), presetsCmd(), testCmd(), statusCmd(), reloadCmd(), bansCmd(), banCmd(), unbanCmd(), versionCmd()) if err := root.Execute(); err != nil { fmt.Fprintln(os.Stderr, "guard:", err) os.Exit(1) @@ -132,8 +133,19 @@ func runCmd() *cobra.Command { if err != nil { return err } + d.Load = load c, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + // SIGHUP reloads, as guard reload does. + hup := make(chan os.Signal, 1) + signal.Notify(hup, syscall.SIGHUP) + go func() { + for range hup { + if _, err := d.ReloadFile(); err != nil { + log.Error("reload refused; the configuration stays as it was", "err", err) + } + } + }() log.Info("starting", "version", version.String(), "config", flagConfig) return d.Run(c) }, @@ -507,6 +519,32 @@ func clip(s string, n int) string { // ---- status +// askDaemon makes a request of the running Guard, on its socket. +func askDaemon(cfg *config.Config, method, path string) (*http.Response, error) { + hc := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{DialContext: func(c context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(c, "unix", cfg.Socket) + }}} + req, err := http.NewRequest(method, "http://guard"+path, nil) + if err != nil { + return nil, err + } + resp, err := hc.Do(req) + switch { + case errors.Is(err, os.ErrNotExist), errors.Is(err, syscall.ECONNREFUSED): + return nil, fmt.Errorf("Guard isn't running (nothing answers on %s)", cfg.Socket) + case errors.Is(err, os.ErrPermission): + return nil, fmt.Errorf("no permission for %s: run this as root or the guard user", cfg.Socket) + case err != nil: + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return nil, fmt.Errorf("Guard's socket %s: %w", cfg.Socket, err) + } + return resp, nil +} + func statusCmd() *cobra.Command { return &cobra.Command{ Use: "status", Short: "The running Guard: its watches, highest bounties, recent bans", Args: cobra.NoArgs, @@ -515,22 +553,9 @@ func statusCmd() *cobra.Command { if err != nil { return err } - hc := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{DialContext: func(c context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(c, "unix", cfg.Socket) - }}} - resp, err := hc.Get("http://guard/status") - switch { - case errors.Is(err, os.ErrNotExist), errors.Is(err, syscall.ECONNREFUSED): - return fmt.Errorf("Guard isn't running (nothing answers on %s)", cfg.Socket) - case errors.Is(err, os.ErrPermission): - return fmt.Errorf("no permission for %s: run this as root or the guard user", cfg.Socket) - case err != nil: - var ue *url.Error - if errors.As(err, &ue) { - err = ue.Err - } - return fmt.Errorf("Guard's socket %s: %w", cfg.Socket, err) + resp, err := askDaemon(cfg, http.MethodGet, "/status") + if err != nil { + return err } defer resp.Body.Close() var s daemon.Status @@ -575,6 +600,50 @@ func statusCmd() *cobra.Command { } } +// ---- reload + +func reloadCmd() *cobra.Command { + return &cobra.Command{ + Use: "reload", Short: "Make the running Guard read guard.yaml again, keeping its bounties", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + // Checked here first, so a mistake is shown before Guard sees it. + cfg, err := load() + if err != nil { + return err + } + resp, err := askDaemon(cfg, http.MethodPost, "/reload") + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + var e struct{ Error string } + json.NewDecoder(resp.Body).Decode(&e) + return fmt.Errorf("reload refused: %s", e.Error) + } + var r daemon.Reloaded + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + return err + } + if flagJSON { + return printJSON(r) + } + if len(r.Changes) == 0 { + fmt.Println("Reloaded: nothing changed") + } else { + fmt.Println("Reloaded:") + for _, c := range r.Changes { + fmt.Println(" " + c) + } + } + if len(r.Restart) > 0 { + fmt.Printf("Changed, but only on a restart (systemctl restart guard): %s\n", strings.Join(r.Restart, ", ")) + } + return nil + }, + } +} + // plural is "1 ban", "2 bans", "2 matches". func plural(n int, what string) string { if n == 1 { diff --git a/deploy/systemd/guard.service b/deploy/systemd/guard.service index eb02303..7e6c481 100644 --- a/deploy/systemd/guard.service +++ b/deploy/systemd/guard.service @@ -9,6 +9,8 @@ Wants=network-online.target [Service] ExecStart=/usr/local/bin/guard --config /etc/guard/guard.yaml run +# systemctl reload guard: guard.yaml again, keeping the bounties. +ExecReload=/usr/local/bin/guard --config /etc/guard/guard.yaml reload Restart=on-failure RestartSec=5s diff --git a/docs/cli.md b/docs/cli.md index 70c8fee..b286766 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -12,6 +12,7 @@ One binary is the daemon and its tools. Every command reads | `guard presets [PRESET]` | the presets, or one's patterns | | `guard test WATCH [FILE\|-] [--lines N] [--all]` | what a watch's patterns find in a log, and whom they would ban ([presets.md](presets.md)) | | `guard status` | the running Guard: each watch's lines, matches, bans and highest bounties, the recent bans and why any failed, and bans waiting for Housecarl. Asks the socket: as root or the guard user | +| `guard reload` | make the running Guard read guard.yaml again, keeping its bounties, and show what changed (as `systemctl reload guard` does; [configuration.md](configuration.md#reloading)). Asks the socket | | `guard bans [--list L]` | the ban list's active bans, from Housecarl | | `guard ban ADDRESS [--list L] [--protocol tcp --port 22]... [--time 1d] [--reason R]` | ban by hand | | `guard unban ADDRESS [--list L]` | lift an address's bans | diff --git a/docs/configuration.md b/docs/configuration.md index 8089181..9d6afea 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -3,8 +3,9 @@ `/etc/guard/guard.yaml` is Guard's whole configuration. `guard config --example` prints a commented one. Unknown settings are an error; `guard check` checks the file, compiles every pattern and asks Housecarl -for the ban lists. Guard reads it when it starts: -`systemctl restart guard` after a change. +for the ban lists. After a change, `systemctl reload guard` (or `guard +reload`) puts it to work without losing what Guard has counted (see +[Reloading](#reloading)). ## housecarl @@ -116,6 +117,29 @@ every 30 seconds, for the time it has left. A ban Housecarl refuses (an address it never bans, a list the token doesn't cover) isn't sent again: `guard status` and the log say why. +## Reloading + +`systemctl reload guard`, `guard reload` or a SIGHUP makes the running +Guard read guard.yaml again. `guard reload` checks the file first and +shows what changed; a file with a mistake is refused, and Guard goes on +as it was. + +* A watch is known by its **name**: one that keeps its name keeps its + bounties and counts, and takes its new patterns, hits, window and ban + at once. If its source changed (another file, journal unit, …), the + source starts again; a file is then followed from its end, as when + Guard starts. +* Watches gone (or disabled) stop; new ones start. +* `never_ban` and `defaults.repeat` apply to the next matches and bans. + Bans already made stay as they are, in Housecarl. +* `housecarl`'s connection (`url`, `socket`, the token, `ca_cert`, + `insecure`), `state`, `socket` and `log` only change on a restart: + `guard reload` says so. `housecarl.list` reloads (it only fills in the + watches' lists). + +Changing `hits` doesn't ban at once: an address over the new hits is +banned on its next match. + ## state, socket, log | Setting | Default | | diff --git a/internal/bounty/bounty.go b/internal/bounty/bounty.go index 4b70647..5639b1e 100644 --- a/internal/bounty/bounty.go +++ b/internal/bounty/bounty.go @@ -46,6 +46,13 @@ func (t *Tracker) Add(p netip.Prefix, at time.Time) (int, bool) { return len(ts), false } +// Set changes the hits and window (a reload); the bounties stay. +func (t *Tracker) Set(hits int, within time.Duration) { + t.mu.Lock() + t.Hits, t.Within = hits, within + t.mu.Unlock() +} + // keep drops the times before from (times are in order). func keep(ts []time.Time, from time.Time) []time.Time { i := 0 diff --git a/internal/config/example.yaml b/internal/config/example.yaml index 53f987d..1ee6c35 100644 --- a/internal/config/example.yaml +++ b/internal/config/example.yaml @@ -1,6 +1,6 @@ -# Guard's configuration (docs/configuration.md). Guard reads it when it -# starts: systemctl restart guard after a change, and 'guard check' to -# check it first. +# Guard's configuration (docs/configuration.md). After a change, +# 'systemctl reload guard' (or 'guard reload') puts it to work and keeps +# the bounties; 'guard check' checks it first. # Housecarl, which does the banning. Give Guard a token for its ban list # alone; on the firewall: diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 396d757..27537c8 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -37,16 +37,20 @@ type Banner interface { // Daemon is a running Guard. type Daemon struct { - cfg *config.Config log *slog.Logger hc Banner + logs source.LogReader state *state.State - never *NeverBan - watches []*Watch started time.Time + ctx context.Context // Run's, for what a reload starts - mu sync.Mutex - groups map[string]int // ban list → IPv6 grouping + mu sync.Mutex + // cfg, never and watches change on a reload. + cfg *config.Config + never *NeverBan + neverStop context.CancelFunc + watches []*Watch + groups map[string]int // ban list → IPv6 grouping // pending are the bans Housecarl hasn't taken yet, by list and group. pending map[string]state.Ban wake chan struct{} @@ -54,6 +58,11 @@ type Daemon struct { // once an hour. refused map[string]time.Time + // Load reads the configuration again for a reload (guard run sets it). + Load func() (*config.Config, error) + // Sources makes a watch's source (tests); nil: from its configuration. + Sources func(*config.Watch) source.Source + // Now and RetryEvery can be changed by tests. Now func() time.Time RetryEvery time.Duration @@ -61,18 +70,30 @@ type Daemon struct { // Watch is one watch at work. type Watch struct { - Cfg *config.Watch - Matcher *match.Matcher Source source.Source Tracker *bounty.Tracker + // mu guards Cfg and Matcher, which a reload replaces. + mu sync.Mutex + Cfg *config.Watch + Matcher *match.Matcher + + stop context.CancelFunc + done chan struct{} + lines, matches, bans atomic.Uint64 } +func (w *Watch) setup() (*config.Watch, *match.Matcher) { + w.mu.Lock() + defer w.mu.Unlock() + return w.Cfg, w.Matcher +} + // New makes a Daemon. hc may be a fake; sources come from the watches' // configuration unless a test sets them (Watch.Source). func New(cfg *config.Config, hc Banner, logs source.LogReader, st *state.State, log *slog.Logger) (*Daemon, error) { - d := &Daemon{cfg: cfg, log: log, hc: hc, state: st, never: NewNeverBan(cfg.NeverBan, log), groups: map[string]int{}, + d := &Daemon{cfg: cfg, log: log, hc: hc, logs: logs, state: st, never: NewNeverBan(cfg.NeverBan, log), groups: map[string]int{}, pending: map[string]state.Ban{}, wake: make(chan struct{}, 1), refused: map[string]time.Time{}, Now: time.Now, RetryEvery: 30 * time.Second} for _, b := range st.Pending { @@ -86,16 +107,7 @@ func New(cfg *config.Config, hc Banner, logs source.LogReader, st *state.State, if err != nil { return nil, fmt.Errorf("watch %s: %v", w.Name, err) } - wt := &Watch{Cfg: w, Matcher: m, Tracker: bounty.New(w.Hits, w.Within.D())} - switch w.Kind() { - case "file": - wt.Source = &source.File{Pattern: w.File, Log: log.With("watch", w.Name)} - case "journal": - wt.Source = &source.Journal{Units: w.Journal.Unit, Identifiers: w.Journal.Identifier, Log: log.With("watch", w.Name)} - case config.SourceHousecarlDrops: - wt.Source = &source.HousecarlDrops{Client: logs, Protocol: w.Protocol, Shadow: w.Shadow, Log: log.With("watch", w.Name)} - } - d.watches = append(d.watches, wt) + d.watches = append(d.watches, &Watch{Cfg: w, Matcher: m, Tracker: bounty.New(w.Hits, w.Within.D())}) } if len(d.watches) == 0 { return nil, errors.New("every watch is disabled: nothing to do") @@ -103,58 +115,113 @@ func New(cfg *config.Config, hc Banner, logs source.LogReader, st *state.State, return d, nil } -// Watches are the watches at work (tests set their sources). -func (d *Daemon) Watches() []*Watch { return d.watches } +// newSource makes a watch's source from its configuration. +func (d *Daemon) newSource(w *config.Watch) source.Source { + if d.Sources != nil { + return d.Sources(w) + } + log := d.log.With("watch", w.Name) + switch w.Kind() { + case "journal": + return &source.Journal{Units: w.Journal.Unit, Identifiers: w.Journal.Identifier, Log: log} + case config.SourceHousecarlDrops: + return &source.HousecarlDrops{Client: d.logs, Protocol: w.Protocol, Shadow: w.Shadow, Log: log} + } + return &source.File{Pattern: w.File, Log: log} +} + +// Watches are the watches at work (tests set their sources before Run). +func (d *Daemon) Watches() []*Watch { + d.mu.Lock() + defer d.mu.Unlock() + return append([]*Watch(nil), d.watches...) +} + +func (d *Daemon) config() *config.Config { + d.mu.Lock() + defer d.mu.Unlock() + return d.cfg +} + +func (d *Daemon) neverBan() *NeverBan { + d.mu.Lock() + defer d.mu.Unlock() + return d.never +} // Run watches until ctx ends. func (d *Daemon) Run(ctx context.Context) error { d.started = d.Now() + d.ctx = ctx d.loadGroups(ctx) - go d.never.Run(ctx) - var wg sync.WaitGroup + d.mu.Lock() + d.startNever() for _, w := range d.watches { - lines := make(chan source.Line, 256) - wg.Add(2) - go func() { - defer wg.Done() - defer close(lines) - for { - err := w.Source.Run(ctx, lines) - if ctx.Err() != nil { - return - } - d.log.Error("the watch's source stopped; starting it again in 10 seconds", "watch", w.Cfg.Name, "err", err) - select { - case <-ctx.Done(): - return - case <-time.After(10 * time.Second): - } - } - }() - go func() { - defer wg.Done() - for l := range lines { - d.Line(ctx, w, l) - } - }() - d.log.Info("watching", "watch", w.Cfg.Name, "source", Describe(w.Cfg), "patterns", len(w.Cfg.Match), "hits", w.Cfg.Hits, "within", w.Cfg.Within.String(), "list", w.Cfg.Ban.List) + d.start(w) } + d.mu.Unlock() + var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() d.sender(ctx) }() - if d.cfg.Socket != "" { + if sock := d.config().Socket; sock != "" { if err := d.serveStatus(ctx); err != nil { - d.log.Warn("no status socket (guard status won't work)", "socket", d.cfg.Socket, "err", err) + d.log.Warn("no status socket (guard status won't work)", "socket", sock, "err", err) } } <-ctx.Done() wg.Wait() + for _, w := range d.Watches() { + <-w.done + } d.savePending() return nil } +// startNever looks never_ban's names up until a reload replaces it +// (d.mu held). +func (d *Daemon) startNever() { + c, stop := context.WithCancel(d.ctx) + d.neverStop = stop + go d.never.Run(c) +} + +// start runs a watch: its source, and its lines through Line, until +// stopped or Run's context ends (d.mu held). +func (d *Daemon) start(w *Watch) { + if w.Source == nil { + w.Source = d.newSource(w.Cfg) + } + ctx, stop := context.WithCancel(d.ctx) + w.stop, w.done = stop, make(chan struct{}) + lines := make(chan source.Line, 256) + src := w.Source + go func() { + defer close(lines) + for { + err := src.Run(ctx, lines) + if ctx.Err() != nil { + return + } + d.log.Error("the watch's source stopped; starting it again in 10 seconds", "watch", w.Cfg.Name, "err", err) + select { + case <-ctx.Done(): + return + case <-time.After(10 * time.Second): + } + } + }() + go func() { + defer close(w.done) + for l := range lines { + d.Line(ctx, w, l) + } + }() + d.log.Info("watching", "watch", w.Cfg.Name, "source", Describe(w.Cfg), "patterns", len(w.Cfg.Match), "hits", w.Cfg.Hits, "within", w.Cfg.Within.String(), "list", w.Cfg.Ban.List) +} + // Describe says where a watch's lines come from. func Describe(w *config.Watch) string { switch w.Kind() { @@ -199,8 +266,9 @@ func Short(w *config.Watch) string { // bounties are counted as the list bans. A list that can't be read now // counts by /64, its default. func (d *Daemon) loadGroups(ctx context.Context) { - for _, w := range d.watches { - list := w.Cfg.Ban.List + for _, w := range d.Watches() { + cfg, _ := w.setup() + list := cfg.Ban.List d.mu.Lock() _, done := d.groups[list] d.mu.Unlock() @@ -238,33 +306,34 @@ func (d *Daemon) group(list string) int { // Line handles one line of a watch. func (d *Daemon) Line(ctx context.Context, w *Watch, l source.Line) { w.lines.Add(1) + cfg, m := w.setup() addr := l.Addr if !addr.IsValid() { var ok bool - if addr, ok = w.Matcher.Match(l.Text); !ok { + if addr, ok = m.Match(l.Text); !ok { return } } w.matches.Add(1) - if why := d.never.Why(addr); why != "" { - d.log.Debug("match, but never banned", "watch", w.Cfg.Name, "address", addr, "why", why) + if why := d.neverBan().Why(addr); why != "" { + d.log.Debug("match, but never banned", "watch", cfg.Name, "address", addr, "why", why) return } - g := match.Group(addr, d.group(w.Cfg.Ban.List)) + g := match.Group(addr, d.group(cfg.Ban.List)) n, reached := w.Tracker.Add(g, l.Time) - d.log.Debug("match", "watch", w.Cfg.Name, "address", addr, "bounty", n, "hits", w.Cfg.Hits, "line", clip(l.Text, 300)) + d.log.Debug("match", "watch", cfg.Name, "address", addr, "bounty", n, "hits", cfg.Hits, "line", clip(l.Text, 300)) if !reached { return } w.bans.Add(1) - d.ban(w, addr, g, l.Text) + d.ban(cfg, addr, g, l.Text) } // BanTime is how long a group's next ban lasts: the watch's time, // times the repeat factor for each earlier ban within forget, up to the // most. func (d *Daemon) BanTime(w *config.Watch, group string) time.Duration { - r := d.cfg.Defaults.Repeat + r := d.config().Defaults.Repeat base := w.Ban.Time.D() n := d.state.Repeats(group, d.Now().Add(-r.Forget.D())) t := time.Duration(float64(base) * math.Pow(r.Factor, float64(n))) @@ -277,12 +346,12 @@ func (d *Daemon) BanTime(w *config.Watch, group string) time.Duration { return t } -func (d *Daemon) ban(w *Watch, addr netip.Addr, g netip.Prefix, line string) { +func (d *Daemon) ban(w *config.Watch, addr netip.Addr, g netip.Prefix, line string) { group := g.String() - t := d.BanTime(w.Cfg, group) - b := state.Ban{Time: d.Now(), Watch: w.Cfg.Name, List: w.Cfg.Ban.List, Address: addr.String(), Group: group, - Protocol: strings.ToLower(w.Cfg.Ban.Protocol), Ports: strings.Join(w.Cfg.Ban.Port, ","), Seconds: int(t / time.Second), Reason: clip(line, 300)} - d.log.Info("bounty reached; banning", "watch", w.Cfg.Name, "address", addr, "list", b.List, "ports", portsText(b), "for", config.FormatDuration(t)) + t := d.BanTime(w, group) + b := state.Ban{Time: d.Now(), Watch: w.Name, List: w.Ban.List, Address: addr.String(), Group: group, + Protocol: strings.ToLower(w.Ban.Protocol), Ports: strings.Join(w.Ban.Port, ","), Seconds: int(t / time.Second), Reason: clip(line, 300)} + d.log.Info("bounty reached; banning", "watch", w.Name, "address", addr, "list", b.List, "ports", portsText(b), "for", config.FormatDuration(t)) d.mu.Lock() d.pending[b.List+" "+b.Group+" "+b.Protocol+" "+b.Ports] = b d.mu.Unlock() @@ -390,10 +459,11 @@ func (d *Daemon) savePending() { p = append(p, b) } d.mu.Unlock() + cfg := d.config() d.state.SetPending(p) - d.state.Forget(d.Now().Add(-d.cfg.Defaults.Repeat.Forget.D())) + d.state.Forget(d.Now().Add(-cfg.Defaults.Repeat.Forget.D())) if err := d.state.Save(); err != nil { - d.log.Warn("saving the state failed", "file", d.cfg.State, "err", err) + d.log.Warn("saving the state failed", "file", cfg.State, "err", err) } } @@ -430,9 +500,10 @@ type WatchStatus struct { // Status reports the watches, their highest bounties and the recent bans. func (d *Daemon) Status() Status { s := Status{Started: d.started} - for _, w := range d.watches { - s.Watches = append(s.Watches, WatchStatus{Name: w.Cfg.Name, Source: Short(w.Cfg), List: w.Cfg.Ban.List, Hits: w.Cfg.Hits, - Within: w.Cfg.Within.String(), Lines: w.lines.Load(), Matches: w.matches.Load(), Bans: w.bans.Load(), Top: w.Tracker.Top(10, d.Now())}) + for _, w := range d.Watches() { + cfg, _ := w.setup() + s.Watches = append(s.Watches, WatchStatus{Name: cfg.Name, Source: Short(cfg), List: cfg.Ban.List, Hits: cfg.Hits, + Within: cfg.Within.String(), Lines: w.lines.Load(), Matches: w.matches.Load(), Bans: w.bans.Load(), Top: w.Tracker.Top(10, d.Now())}) } s.Recent, s.Pending = d.state.Snapshot() if len(s.Recent) > 50 { @@ -444,7 +515,7 @@ func (d *Daemon) Status() Status { // serveStatus answers GET /status on the local socket (root and the // guard user only: the socket is 0660 in /run/guard). func (d *Daemon) serveStatus(ctx context.Context) error { - path := d.cfg.Socket + path := d.config().Socket if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return err } @@ -459,6 +530,16 @@ func (d *Daemon) serveStatus(ctx context.Context) error { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(d.Status()) }) + mux.HandleFunc("POST /reload", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + res, err := d.ReloadFile() + if err != nil { + w.WriteHeader(http.StatusUnprocessableEntity) + json.NewEncoder(w).Encode(map[string]string{"error": err.Error()}) + return + } + json.NewEncoder(w).Encode(res) + }) srv := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} go srv.Serve(ln) go func() { diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 4416e22..35f0989 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -177,3 +177,140 @@ socket: ` + filepath.Join(dir, "guard.sock") + ` t.Errorf("saved state: %v %+v", err, st2.Offences) } } + +// stoppable is a chanSource that says when it stopped. +type stoppable struct { + chanSource + stopped chan struct{} +} + +func (s *stoppable) Run(ctx context.Context, out chan<- source.Line) error { + err := s.chanSource.Run(ctx, out) + close(s.stopped) + return err +} + +// TestReload: a reload keeps the bounties of a watch under the same name +// and applies its new settings; a watch whose source changed gets a new +// one; watches gone stop, new ones start; settings for a restart are +// reported; a configuration with nothing to watch is refused. +func TestReload(t *testing.T) { + dir := t.TempDir() + base := `housecarl: {url: https://fw:8070, list: web} +defaults: {hits: 3, within: 10m, ban: 1h} +state: ` + filepath.Join(dir, "state.json") + ` +socket: ` + filepath.Join(dir, "guard.sock") + ` +` + parse := func(extra string) *config.Config { + t.Helper() + cfg, err := config.Parse([]byte(base + extra)) + if err != nil { + t.Fatal(err) + } + return cfg + } + cfg := parse(`watch: + - {name: ssh, preset: sshd, file: /var/log/auth.log} + - {name: web, file: /var/log/web.log, match: ['bad $']} +`) + st, _ := state.Load(cfg.State) + hc := &fakeHC{} + d, err := New(cfg, hc, nil, st, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + srcs := map[string]*stoppable{} + d.Sources = func(w *config.Watch) source.Source { + mu.Lock() + defer mu.Unlock() + s := &stoppable{chanSource{make(chan source.Line)}, make(chan struct{})} + srcs[w.Name+" "+w.File] = s + return s + } + src := func(key string) *stoppable { + mu.Lock() + defer mu.Unlock() + return srcs[key] + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { d.Run(ctx); close(done) }() + waitFor(t, "the sources", func() bool { return src("ssh /var/log/auth.log") != nil && src("web /var/log/web.log") != nil }) + fail := func(s *stoppable, addr string, n int) { + for i := 0; i < n; i++ { + s.ch <- source.Line{Text: "Failed password for root from " + addr + " port 4000 ssh2", Time: time.Now()} + } + } + ssh := src("ssh /var/log/auth.log") + fail(ssh, "203.0.113.9", 2) + + res, err := d.Reload(parse(`never_ban: [198.51.100.0/24] +watch: + - {name: ssh, preset: sshd, file: /var/log/auth.log, ban: {time: 2h}} + - {name: mail, preset: dovecot, file: /var/log/mail.log} +`)) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(res.Changes, "; "); got != "never_ban; watch ssh: ban; watch mail: added; watch web: removed" || len(res.Restart) != 0 { + t.Errorf("reload: %q, restart %v", got, res.Restart) + } + select { + case <-src("web /var/log/web.log").stopped: + case <-time.After(2 * time.Second): + t.Error("the removed watch's source still runs") + } + if src("mail /var/log/mail.log") == nil { + t.Error("the added watch has no source") + } + // The bounty of 2 was kept: one more bans, for the new time. + fail(ssh, "203.0.113.9", 1) + waitFor(t, "the ban after the reload", func() bool { return hc.count() == 1 }) + if b := hc.last(); b.Time < 7190 || b.Time > 7200 { + t.Errorf("ban after reload: %ds, want 2h", b.Time) + } + fail(ssh, "198.51.100.7", 3) + + // A new file: a new source; the bounty stays; the hits change. + fail(ssh, "203.0.113.50", 2) + res, err = d.Reload(parse(`log: {level: debug} +watch: + - {name: ssh, preset: sshd, file: /var/log/secure, hits: 4, ban: {time: 2h}} +`)) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(res.Changes, "; "); got != "never_ban; watch ssh: bounty 4 in 10m, source (started again); watch mail: removed" || + strings.Join(res.Restart, ",") != "log" { + t.Errorf("reload: %q, restart %v", got, res.Restart) + } + select { + case <-ssh.stopped: + case <-time.After(2 * time.Second): + t.Error("the old source still runs") + } + secure := src("ssh /var/log/secure") + fail(secure, "203.0.113.50", 1) + time.Sleep(50 * time.Millisecond) + if hc.count() != 1 { + t.Fatal("banned below the new hits") + } + fail(secure, "203.0.113.50", 1) + waitFor(t, "the ban at 4", func() bool { return hc.count() == 2 && hc.last().Address == "203.0.113.50" }) + if d.config().Log.Level != "info" { + t.Error("the log level changed without a restart") + } + + // Nothing to watch: refused, the old configuration stays. + if _, err := d.Reload(parse(`watch: + - {name: ssh, preset: sshd, disabled: true} +`)); err == nil { + t.Error("a configuration with every watch disabled was taken") + } + if s := d.Status(); len(s.Watches) != 1 || s.Watches[0].Hits != 4 || s.Watches[0].Bans != 2 || s.Watches[0].Lines != 10 { + t.Errorf("status after reloads %+v", s.Watches) + } + cancel() + <-done +} diff --git a/internal/daemon/reload.go b/internal/daemon/reload.go new file mode 100644 index 0000000..b7e50c7 --- /dev/null +++ b/internal/daemon/reload.go @@ -0,0 +1,185 @@ +package daemon + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "guard/internal/bounty" + "guard/internal/config" + "guard/internal/match" +) + +// Reloaded is what a reload changed. +type Reloaded struct { + Changes []string `json:"changes"` + // Restart are the changed settings a reload can't apply: they wait + // for a restart. + Restart []string `json:"restart,omitempty"` +} + +// ReloadFile reads the configuration again (Load) and reloads it. +func (d *Daemon) ReloadFile() (*Reloaded, error) { + if d.Load == nil { + return nil, errors.New("this Guard can't read its configuration again") + } + cfg, err := d.Load() + if err != nil { + return nil, err + } + return d.Reload(cfg) +} + +// Reload puts a new configuration to work without losing what Guard +// counted: a watch keeps its bounties and counts under the same name, +// and its source runs on unless it changed. Watches gone stop, new ones +// start. The housecarl connection, state, socket and log wait for a +// restart. A configuration with no watch at work is refused, and the +// old one stays. +func (d *Daemon) Reload(cfg *config.Config) (*Reloaded, error) { + if d.ctx == nil { + return nil, errors.New("Guard isn't running") + } + type next struct { + cfg *config.Watch + m *match.Matcher + } + var want []next + for _, w := range cfg.Watch { + if w.Disabled { + continue + } + m, err := match.Compile(w.Match, w.Ignore) + if err != nil { + return nil, fmt.Errorf("watch %s: %v", w.Name, err) + } + want = append(want, next{w, m}) + } + if len(want) == 0 { + return nil, errors.New("every watch is disabled: nothing to do (the configuration stays as it was)") + } + + res := &Reloaded{} + d.mu.Lock() + old := d.cfg + oh, nh := old.Housecarl, cfg.Housecarl + oh.List, nh.List = "", "" // the default list only fills in watches + if oh != nh { + res.Restart = append(res.Restart, "housecarl") + } + if old.State != cfg.State { + res.Restart = append(res.Restart, "state") + } + if old.Socket != cfg.Socket { + res.Restart = append(res.Restart, "socket") + } + if old.Log != cfg.Log { + res.Restart = append(res.Restart, "log") + } + cfg.Housecarl.URL, cfg.Housecarl.Socket, cfg.Housecarl.TokenFile, cfg.Housecarl.Token = old.Housecarl.URL, old.Housecarl.Socket, old.Housecarl.TokenFile, old.Housecarl.Token + cfg.Housecarl.CACert, cfg.Housecarl.Insecure = old.Housecarl.CACert, old.Housecarl.Insecure + cfg.State, cfg.Socket, cfg.Log = old.State, old.Socket, old.Log + + if !slices.Equal(old.NeverBan, cfg.NeverBan) { + n := NewNeverBan(cfg.NeverBan, d.log) + n.Lookup = d.never.Lookup + d.never.mu.RLock() + for _, h := range n.names { + if a, ok := d.never.resolved[h]; ok { + n.resolved[h] = a + } + } + d.never.mu.RUnlock() + d.neverStop() + d.never = n + d.startNever() + res.Changes = append(res.Changes, "never_ban") + } + if old.Defaults.Repeat != cfg.Defaults.Repeat { + res.Changes = append(res.Changes, "defaults.repeat: for the next bans") + } + + byName := map[string]*Watch{} + for _, w := range d.watches { + byName[strings.ToLower(w.Cfg.Name)] = w + } + var watches []*Watch + var stopped []chan struct{} + for _, n := range want { + w := byName[strings.ToLower(n.cfg.Name)] + delete(byName, strings.ToLower(n.cfg.Name)) + if w == nil { + w = &Watch{Cfg: n.cfg, Matcher: n.m, Tracker: bounty.New(n.cfg.Hits, n.cfg.Within.D())} + d.start(w) + watches = append(watches, w) + res.Changes = append(res.Changes, "watch "+n.cfg.Name+": added") + continue + } + watches = append(watches, w) + oc, _ := w.setup() + if reflect.DeepEqual(oc, n.cfg) { + continue + } + var what []string + if !slices.Equal(oc.Match, n.cfg.Match) || !slices.Equal(oc.Ignore, n.cfg.Ignore) { + what = append(what, "patterns") + } + if oc.Hits != n.cfg.Hits || oc.Within != n.cfg.Within { + what = append(what, fmt.Sprintf("bounty %d in %s", n.cfg.Hits, n.cfg.Within)) + } + if !reflect.DeepEqual(oc.Ban, n.cfg.Ban) { + what = append(what, "ban") + } + w.Tracker.Set(n.cfg.Hits, n.cfg.Within.D()) + w.mu.Lock() + w.Cfg, w.Matcher = n.cfg, n.m + w.mu.Unlock() + if sourceKey(oc) != sourceKey(n.cfg) { + w.stop() + stopped = append(stopped, w.done) + w.Source = d.newSource(n.cfg) + d.start(w) + what = append(what, "source (started again)") + } + if len(what) == 0 { + what = append(what, "changed") + } + res.Changes = append(res.Changes, "watch "+n.cfg.Name+": "+strings.Join(what, ", ")) + } + for _, w := range d.watches { + if byName[strings.ToLower(w.Cfg.Name)] == w { + w.stop() + stopped = append(stopped, w.done) + res.Changes = append(res.Changes, "watch "+w.Cfg.Name+": removed") + } + } + d.watches = watches + d.cfg = cfg + d.mu.Unlock() + + // A stopped watch may be handling a line, which needs d.mu. + for _, done := range stopped { + <-done + } + d.loadGroups(d.ctx) + if len(res.Changes) == 0 { + res.Changes = []string{} + } + d.log.Info("reloaded", "changes", strings.Join(res.Changes, "; ")) + if len(res.Restart) > 0 { + d.log.Warn("changed settings that wait for a restart", "settings", strings.Join(res.Restart, ", ")) + } + return res, nil +} + +// sourceKey is what a watch's source is made from: when it changes, the +// source starts again. +func sourceKey(w *config.Watch) string { + j := "" + if w.Journal != nil { + j = fmt.Sprint(w.Journal.Unit, w.Journal.Identifier) + } + return fmt.Sprint(w.Kind(), "|", w.File, "|", j, "|", w.Source, "|", w.Protocol, "|", w.Shadow) +}