Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 47 additions & 17 deletions cmd/fluent-watcher/fluentbit/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
defaultParserCfgPath = "/fluent-bit/etc/parser.conf"
defaultWatchDir = "/fluent-bit/config"
defaultPollInterval = 1 * time.Second
defaultReloadDelay = 1 * time.Second
)

func main() {
Expand All @@ -36,12 +37,15 @@ func main() {
var poll bool
var pollInterval time.Duration
var parserCfgPath string
var reloadDelay time.Duration
flag.StringVar(&binPath, "b", defaultBinPath, "The fluent bit binary path.")
flag.StringVar(&externalPluginPath, "e", "", "Path to external plugin (shared lib)")
flag.StringVar(&watchPath, "watch-path", defaultWatchDir, "The path to watch.")
flag.BoolVar(&poll, "poll", false, "Use poll watcher instead of ionotify.")
flag.DurationVar(&pollInterval, "poll-interval", defaultPollInterval, "Poll interval if using poll watcher.")
flag.StringVar(&parserCfgPath, "R", defaultParserCfgPath, "Specify a parser config file")
flag.DurationVar(&reloadDelay, "reload-delay", defaultReloadDelay,
"How long the watch path has to stay unchanged before the config is reloaded.")

// Deprecated flags to be removed in one of the next releases.
var exitOnFailure bool
Expand Down Expand Up @@ -119,24 +123,13 @@ func main() {
return fmt.Errorf("failed to watch path %q: %w", watchPath, err)
}

for {
select {
case <-signalCtx.Done():
return nil
case <-grpCtx.Done():
return nil
case event := <-watcher.Events():
if !isValidEvent(event) {
continue
}
_ = level.Info(l).Log("msg", "Config file changed, reloading...")
if err := cmd.Process.Signal(syscall.SIGHUP); err != nil {
return fmt.Errorf("failed to reload config: %w", err)
}
case err := <-watcher.Errors():
return fmt.Errorf("failed the watcher: %w", err)
return watchAndReload(signalCtx, grpCtx, watcher, reloadDelay, func() error {
_ = level.Info(l).Log("msg", "Config file changed, reloading...")
if err := cmd.Process.Signal(syscall.SIGHUP); err != nil {
return fmt.Errorf("failed to reload config: %w", err)
}
}
return nil
})
})

select {
Expand All @@ -157,6 +150,43 @@ func main() {
}
}

// watchAndReload calls reload once the watched path has been quiet for delay after a change.
//
// A single update of a mounted Secret or ConfigMap is a burst of events: the kubelet creates a
// new timestamped directory, writes the files into it and only then switches the "..data"
// symlink. Reloading on the first event makes fluent-bit read the config before the switch,
// so it loads the previous version, and it drops the signals that follow while that reload
// is running. Waiting for the burst to end gives one reload per update, on the new content.
func watchAndReload(
signalCtx, grpCtx context.Context,
watcher filenotify.FileWatcher,
delay time.Duration,
reload func() error,
) error {
var pending <-chan time.Time
for {
select {
case <-signalCtx.Done():
return nil
case <-grpCtx.Done():
return nil
case event := <-watcher.Events():
if !isValidEvent(event) {
continue
}
// A new timer per event: the delay counts from the last change.
pending = time.After(delay)
case <-pending:
pending = nil
if err := reload(); err != nil {
return err
}
case err := <-watcher.Errors():
return fmt.Errorf("failed the watcher: %w", err)
}
}
}

// Inspired by https://github.com/jimmidyson/configmap-reload
func isValidEvent(event fsnotify.Event) bool {
return event.Op == fsnotify.Create || event.Op == fsnotify.Write
Expand Down
147 changes: 147 additions & 0 deletions cmd/fluent-watcher/fluentbit/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package main

import (
"context"
"os"
"path/filepath"
"testing"
"time"

"github.com/fluent/fluent-operator/v3/pkg/filenotify"
)

const testReloadDelay = 200 * time.Millisecond

// startWatch runs watchAndReload on dir. Each reload sends the content the config file has at
// that moment, read through the "..data" symlink like fluent-bit reads it.
func startWatch(t *testing.T, dir string) <-chan string {
t.Helper()

watcher, err := filenotify.NewEventWatcher()
if err != nil {
t.Fatalf("failed to create the watcher: %v", err)
}
if err := watcher.Add(dir); err != nil {
t.Fatalf("failed to watch %q: %v", dir, err)
}

ctx, cancel := context.WithCancel(context.Background())
reloads := make(chan string, 16)
done := make(chan struct{})
go func() {
defer close(done)
_ = watchAndReload(ctx, ctx, watcher, testReloadDelay, func() error {
content, err := os.ReadFile(filepath.Join(dir, "fluent-bit.yaml"))
if err != nil {
return err
}
reloads <- string(content)
return nil
})
}()
t.Cleanup(func() {
cancel()
<-done
_ = watcher.Close()
})

return reloads
}

// writeVolume updates dir the way the kubelet updates a Secret volume: a new timestamped
// directory, the payload, then an atomic switch of the "..data" symlink, then removal of the
// old directory. pause is slept between creating the directory and switching the symlink.
func writeVolume(t *testing.T, dir, version, content string, pause time.Duration) {
t.Helper()

tsDir := filepath.Join(dir, ".."+version)
if err := os.Mkdir(tsDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(tsDir, "fluent-bit.yaml"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
time.Sleep(pause)

dataLink := filepath.Join(dir, "..data")
oldTsDir, _ := os.Readlink(dataLink)
tmpLink := filepath.Join(dir, "..data_tmp")
if err := os.Symlink(filepath.Base(tsDir), tmpLink); err != nil {
t.Fatal(err)
}
if err := os.Rename(tmpLink, dataLink); err != nil {
t.Fatal(err)
}

visible := filepath.Join(dir, "fluent-bit.yaml")
if _, err := os.Lstat(visible); os.IsNotExist(err) {
if err := os.Symlink(filepath.Join("..data", "fluent-bit.yaml"), visible); err != nil {
t.Fatal(err)
}
}
if oldTsDir != "" {
if err := os.RemoveAll(filepath.Join(dir, oldTsDir)); err != nil {
t.Fatal(err)
}
}
}

func expectReload(t *testing.T, reloads <-chan string, want string) {
t.Helper()
select {
case got := <-reloads:
if got != want {
t.Fatalf("reloaded config %q, want %q", got, want)
}
case <-time.After(10 * testReloadDelay):
t.Fatalf("no reload for config %q", want)
}
}

func expectNoReload(t *testing.T, reloads <-chan string) {
t.Helper()
select {
case got := <-reloads:
t.Fatalf("unexpected extra reload with config %q", got)
case <-time.After(3 * testReloadDelay):
}
}

func TestWatchAndReloadReadsTheSwitchedConfig(t *testing.T) {
dir := t.TempDir()
writeVolume(t, dir, "v1", "v1", 0)
reloads := startWatch(t, dir)

// The pause keeps the new directory visible before "..data" points to it, which is the
// window in which a reload on the first event reads the previous config.
writeVolume(t, dir, "v2", "v2", testReloadDelay/4)

expectReload(t, reloads, "v2")
expectNoReload(t, reloads)
}

func TestWatchAndReloadCoalescesUpdatesInARow(t *testing.T) {
dir := t.TempDir()
writeVolume(t, dir, "v1", "v1", 0)
reloads := startWatch(t, dir)

writeVolume(t, dir, "v2", "v2", 0)
time.Sleep(testReloadDelay / 2)
writeVolume(t, dir, "v3", "v3", 0)

expectReload(t, reloads, "v3")
expectNoReload(t, reloads)
}

func TestWatchAndReloadReloadsAgainAfterALaterUpdate(t *testing.T) {
dir := t.TempDir()
writeVolume(t, dir, "v1", "v1", 0)
reloads := startWatch(t, dir)

writeVolume(t, dir, "v2", "v2", 0)
expectReload(t, reloads, "v2")

writeVolume(t, dir, "v3", "v3", 0)
expectReload(t, reloads, "v3")
expectNoReload(t, reloads)
}
Loading