From a51ab26dc13d0d05b31aa9a1adb86f8b93095621 Mon Sep 17 00:00:00 2001 From: Siarei Rasiukevich Date: Fri, 13 Nov 2026 13:25:40 +0200 Subject: [PATCH] fix(fluent-watcher): reload fluent-bit after the config volume is switched The watcher sent SIGHUP on the first file event in the watch path. One update of a mounted Secret is a burst of events: the kubelet creates a new timestamped directory, writes the files and only then switches the "..data" symlink. The first event comes from the new directory, so fluent-bit started the hot reload before the switch and read the previous config. The events of the switch itself arrived while that reload was running and were dropped ("reloading in progress, aborting"), so the pod stayed one config version behind until the next change, while reporting a successful reload. Wait until the watch path has been unchanged for -reload-delay (default 1s) and send one SIGHUP per update. Signed-off-by: Siarei Rasiukevich --- cmd/fluent-watcher/fluentbit/main.go | 64 +++++++--- cmd/fluent-watcher/fluentbit/main_test.go | 147 ++++++++++++++++++++++ 2 files changed, 194 insertions(+), 17 deletions(-) create mode 100644 cmd/fluent-watcher/fluentbit/main_test.go diff --git a/cmd/fluent-watcher/fluentbit/main.go b/cmd/fluent-watcher/fluentbit/main.go index af9da18bb..e7d0b6797 100644 --- a/cmd/fluent-watcher/fluentbit/main.go +++ b/cmd/fluent-watcher/fluentbit/main.go @@ -26,6 +26,7 @@ const ( defaultParserCfgPath = "/fluent-bit/etc/parser.conf" defaultWatchDir = "/fluent-bit/config" defaultPollInterval = 1 * time.Second + defaultReloadDelay = 1 * time.Second ) func main() { @@ -36,12 +37,15 @@ func main() { var poll bool var pollInterval time.Duration var parserCfgPath string + var reloadDelay time.Duration flag.StringVar(&binPath, "b", defaultBinPath, "The fluent bit binary path.") flag.StringVar(&externalPluginPath, "e", "", "Path to external plugin (shared lib)") flag.StringVar(&watchPath, "watch-path", defaultWatchDir, "The path to watch.") flag.BoolVar(&poll, "poll", false, "Use poll watcher instead of ionotify.") flag.DurationVar(&pollInterval, "poll-interval", defaultPollInterval, "Poll interval if using poll watcher.") flag.StringVar(&parserCfgPath, "R", defaultParserCfgPath, "Specify a parser config file") + flag.DurationVar(&reloadDelay, "reload-delay", defaultReloadDelay, + "How long the watch path has to stay unchanged before the config is reloaded.") // Deprecated flags to be removed in one of the next releases. var exitOnFailure bool @@ -119,24 +123,13 @@ func main() { return fmt.Errorf("failed to watch path %q: %w", watchPath, err) } - for { - select { - case <-signalCtx.Done(): - return nil - case <-grpCtx.Done(): - return nil - case event := <-watcher.Events(): - if !isValidEvent(event) { - continue - } - _ = level.Info(l).Log("msg", "Config file changed, reloading...") - if err := cmd.Process.Signal(syscall.SIGHUP); err != nil { - return fmt.Errorf("failed to reload config: %w", err) - } - case err := <-watcher.Errors(): - return fmt.Errorf("failed the watcher: %w", err) + return watchAndReload(signalCtx, grpCtx, watcher, reloadDelay, func() error { + _ = level.Info(l).Log("msg", "Config file changed, reloading...") + if err := cmd.Process.Signal(syscall.SIGHUP); err != nil { + return fmt.Errorf("failed to reload config: %w", err) } - } + return nil + }) }) select { @@ -157,6 +150,43 @@ func main() { } } +// watchAndReload calls reload once the watched path has been quiet for delay after a change. +// +// A single update of a mounted Secret or ConfigMap is a burst of events: the kubelet creates a +// new timestamped directory, writes the files into it and only then switches the "..data" +// symlink. Reloading on the first event makes fluent-bit read the config before the switch, +// so it loads the previous version, and it drops the signals that follow while that reload +// is running. Waiting for the burst to end gives one reload per update, on the new content. +func watchAndReload( + signalCtx, grpCtx context.Context, + watcher filenotify.FileWatcher, + delay time.Duration, + reload func() error, +) error { + var pending <-chan time.Time + for { + select { + case <-signalCtx.Done(): + return nil + case <-grpCtx.Done(): + return nil + case event := <-watcher.Events(): + if !isValidEvent(event) { + continue + } + // A new timer per event: the delay counts from the last change. + pending = time.After(delay) + case <-pending: + pending = nil + if err := reload(); err != nil { + return err + } + case err := <-watcher.Errors(): + return fmt.Errorf("failed the watcher: %w", err) + } + } +} + // Inspired by https://github.com/jimmidyson/configmap-reload func isValidEvent(event fsnotify.Event) bool { return event.Op == fsnotify.Create || event.Op == fsnotify.Write diff --git a/cmd/fluent-watcher/fluentbit/main_test.go b/cmd/fluent-watcher/fluentbit/main_test.go new file mode 100644 index 000000000..21bfaad54 --- /dev/null +++ b/cmd/fluent-watcher/fluentbit/main_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/fluent/fluent-operator/v3/pkg/filenotify" +) + +const testReloadDelay = 200 * time.Millisecond + +// startWatch runs watchAndReload on dir. Each reload sends the content the config file has at +// that moment, read through the "..data" symlink like fluent-bit reads it. +func startWatch(t *testing.T, dir string) <-chan string { + t.Helper() + + watcher, err := filenotify.NewEventWatcher() + if err != nil { + t.Fatalf("failed to create the watcher: %v", err) + } + if err := watcher.Add(dir); err != nil { + t.Fatalf("failed to watch %q: %v", dir, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + reloads := make(chan string, 16) + done := make(chan struct{}) + go func() { + defer close(done) + _ = watchAndReload(ctx, ctx, watcher, testReloadDelay, func() error { + content, err := os.ReadFile(filepath.Join(dir, "fluent-bit.yaml")) + if err != nil { + return err + } + reloads <- string(content) + return nil + }) + }() + t.Cleanup(func() { + cancel() + <-done + _ = watcher.Close() + }) + + return reloads +} + +// writeVolume updates dir the way the kubelet updates a Secret volume: a new timestamped +// directory, the payload, then an atomic switch of the "..data" symlink, then removal of the +// old directory. pause is slept between creating the directory and switching the symlink. +func writeVolume(t *testing.T, dir, version, content string, pause time.Duration) { + t.Helper() + + tsDir := filepath.Join(dir, ".."+version) + if err := os.Mkdir(tsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tsDir, "fluent-bit.yaml"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + time.Sleep(pause) + + dataLink := filepath.Join(dir, "..data") + oldTsDir, _ := os.Readlink(dataLink) + tmpLink := filepath.Join(dir, "..data_tmp") + if err := os.Symlink(filepath.Base(tsDir), tmpLink); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmpLink, dataLink); err != nil { + t.Fatal(err) + } + + visible := filepath.Join(dir, "fluent-bit.yaml") + if _, err := os.Lstat(visible); os.IsNotExist(err) { + if err := os.Symlink(filepath.Join("..data", "fluent-bit.yaml"), visible); err != nil { + t.Fatal(err) + } + } + if oldTsDir != "" { + if err := os.RemoveAll(filepath.Join(dir, oldTsDir)); err != nil { + t.Fatal(err) + } + } +} + +func expectReload(t *testing.T, reloads <-chan string, want string) { + t.Helper() + select { + case got := <-reloads: + if got != want { + t.Fatalf("reloaded config %q, want %q", got, want) + } + case <-time.After(10 * testReloadDelay): + t.Fatalf("no reload for config %q", want) + } +} + +func expectNoReload(t *testing.T, reloads <-chan string) { + t.Helper() + select { + case got := <-reloads: + t.Fatalf("unexpected extra reload with config %q", got) + case <-time.After(3 * testReloadDelay): + } +} + +func TestWatchAndReloadReadsTheSwitchedConfig(t *testing.T) { + dir := t.TempDir() + writeVolume(t, dir, "v1", "v1", 0) + reloads := startWatch(t, dir) + + // The pause keeps the new directory visible before "..data" points to it, which is the + // window in which a reload on the first event reads the previous config. + writeVolume(t, dir, "v2", "v2", testReloadDelay/4) + + expectReload(t, reloads, "v2") + expectNoReload(t, reloads) +} + +func TestWatchAndReloadCoalescesUpdatesInARow(t *testing.T) { + dir := t.TempDir() + writeVolume(t, dir, "v1", "v1", 0) + reloads := startWatch(t, dir) + + writeVolume(t, dir, "v2", "v2", 0) + time.Sleep(testReloadDelay / 2) + writeVolume(t, dir, "v3", "v3", 0) + + expectReload(t, reloads, "v3") + expectNoReload(t, reloads) +} + +func TestWatchAndReloadReloadsAgainAfterALaterUpdate(t *testing.T) { + dir := t.TempDir() + writeVolume(t, dir, "v1", "v1", 0) + reloads := startWatch(t, dir) + + writeVolume(t, dir, "v2", "v2", 0) + expectReload(t, reloads, "v2") + + writeVolume(t, dir, "v3", "v3", 0) + expectReload(t, reloads, "v3") + expectNoReload(t, reloads) +}