*: Change the engine behind ContentPathReloader to be completely independent of any filesystem concept.

cmd/thanos/receive.go

@@ -226,7 +226,7 @@ func runReceive(
 			return errors.Wrap(err, "parse limit configuration")
 		}
 	}
-	limiter, err := receive.NewLimiter(conf.writeLimitsConfig, reg, receiveMode, log.With(logger, "component", "receive-limiter"))
+	limiter, err := receive.NewLimiter(conf.writeLimitsConfig, reg, receiveMode, log.With(logger, "component", "receive-limiter"), conf.limitsConfigReloadTimer)
 	if err != nil {
 		return errors.Wrap(err, "creating limiter")
 	}
@@ -822,8 +822,9 @@ type receiveConfig struct {
 	reqLogConfig      *extflag.PathOrContent
 	relabelConfigPath *extflag.PathOrContent
 
-	writeLimitsConfig *extflag.PathOrContent
-	storeRateLimits   store.SeriesSelectLimits
+	writeLimitsConfig       *extflag.PathOrContent
+	storeRateLimits         store.SeriesSelectLimits
+	limitsConfigReloadTimer time.Duration
 }
 
 func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
@@ -953,6 +954,8 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
 	rc.reqLogConfig = extkingpin.RegisterRequestLoggingFlags(cmd)
 
 	rc.writeLimitsConfig = extflag.RegisterPathOrContent(cmd, "receive.limits-config", "YAML file that contains limit configuration.", extflag.WithEnvSubstitution(), extflag.WithHidden())
+	cmd.Flag("receive.limits-config-reload-timer", "Minimum amount of time to pass for the limit configuration to be reloaded. Helps to avoid excessive reloads.").
+		Default("1s").Hidden().DurationVar(&rc.limitsConfigReloadTimer)
 }
 
 // determineMode returns the ReceiverMode that this receiver is configured to run in.

pkg/extkingpin/path_content_reloader.go

@@ -5,6 +5,7 @@ package extkingpin
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"os"
 	"path"
@@ -22,32 +23,125 @@ type fileContent interface {
 	Path() string
 }
 
-// PathContentReloader starts a file watcher that monitors the file indicated by fileContent.Path() and runs
-// reloadFunc whenever a change is detected.
-// A debounce timer can be configured via opts to handle situations where many "write" events are received together or
-// a "create" event is followed up by a "write" event, for example. Files will be effectively reloaded at the latest
-// after 2 times the debounce timer. By default the debouncer timer is 1 second.
-// To ensure renames and deletes are properly handled, the file watcher is put at the file's parent folder. See
-// https://github.com/fsnotify/fsnotify/issues/214 for more details.
+// PathContentReloader runs the reloadFunc when it detects that the contents of fileContent have changed.
 func PathContentReloader(ctx context.Context, fileContent fileContent, logger log.Logger, reloadFunc func(), debounceTime time.Duration) error {
 	filePath, err := filepath.Abs(fileContent.Path())
 	if err != nil {
 		return errors.Wrap(err, "getting absolute file path")
 	}
-
-	watcher, err := fsnotify.NewWatcher()
 	if filePath == "" {
 		level.Debug(logger).Log("msg", "no path detected for config reload")
 	}
+
+	// Check if filePath is symlink
+	filePathStat, err := os.Lstat(filePath)
+	if err != nil {
+		return errors.Wrap(err, "getting file info")
+	}
+	// Check if filePath's parent folder is symlink
+	parentFolder := path.Dir(filePath)
+	parentFolderStat, err := os.Lstat(parentFolder)
+	if err != nil {
+		return errors.Wrap(err, "getting parent folder info")
+	}
+	var engine reloaderEngine
+	if filePathStat.Mode()&os.ModeSymlink != 0 || parentFolderStat.Mode()&os.ModeSymlink != 0 {
+		level.Debug(logger).Log("msg", "file is a symlink, using polling approach", "file", filePath)
+		engine = &pollingEngine{
+			filePath:   filePath,
+			logger:     logger,
+			debounce:   debounceTime,
+			reloadFunc: reloadFunc,
+		}
+	} else {
+		engine = &fsNotifyEngine{
+			filePath:   filePath,
+			logger:     logger,
+			debounce:   debounceTime,
+			reloadFunc: reloadFunc,
+		}
+	}
+	return engine.Start(ctx)
+}
+
+// reloaderEngine is an interface that abstracts different underlying logics for reloading a `fileContent`.
+type reloaderEngine interface {
+	Start(ctx context.Context) error
+}
+
+// pollingEngine is an implementation of reloaderEngine that keeps rereading the contents at filePath and when the
+// checksum changes it runs the reloadFunc.
+type pollingEngine struct {
+	filePath         string
+	logger           log.Logger
+	debounce         time.Duration
+	reloadFunc       func()
+	previousChecksum [sha256.Size]byte
+}
+
+var _ reloaderEngine = &pollingEngine{}
+
+func (p *pollingEngine) Start(ctx context.Context) error {
+	configReader := func() {
+		// check if file still exists
+		if _, err := os.Stat(p.filePath); os.IsNotExist(err) {
+			level.Error(p.logger).Log("msg", "file does not exist", "error", err)
+			return
+		}
+		file, err := os.ReadFile(p.filePath)
+		if err != nil {
+			level.Error(p.logger).Log("msg", "error opening file", "error", err)
+			return
+		}
+		checksum := sha256.Sum256(file)
+		if checksum == p.previousChecksum {
+			return
+		}
+		p.reloadFunc()
+		p.previousChecksum = checksum
+		level.Debug(p.logger).Log("msg", "configuration reloaded")
+		return
+	}
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(p.debounce):
+				configReader()
+			}
+		}
+	}()
+	return nil
+}
+
+// fsNotifyEngine is an implementation of reloaderEngine that uses fsnotify to watch for changes to a file and then
+// runs the reloadFunc.
+// A debounce timer can be configured via opts to handle situations where many "write" events are received together or
+// a "create" event is followed up by a "write" event, for example. Files will be effectively reloaded at the latest
+// after 2 times the debounce timer. By default the debouncer timer is 1 second.
+// To ensure renames and deletes are properly handled, the file watcher is put at the file's parent folder. See
+// https://github.com/fsnotify/fsnotify/issues/214 for more details.
+type fsNotifyEngine struct {
+	filePath   string
+	logger     log.Logger
+	debounce   time.Duration
+	reloadFunc func()
+}
+
+var _ reloaderEngine = &fsNotifyEngine{}
+
+func (f *fsNotifyEngine) Start(ctx context.Context) error {
+	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return errors.Wrap(err, "creating file watcher")
 	}
 	go func() {
 		var reloadTimer *time.Timer
-		if debounceTime != 0 {
-			reloadTimer = time.AfterFunc(debounceTime, func() {
-				reloadFunc()
-				level.Debug(logger).Log("msg", "configuration reloaded after debouncing")
+		if f.debounce != 0 {
+			reloadTimer = time.AfterFunc(f.debounce, func() {
+				f.reloadFunc()
+				level.Debug(f.logger).Log("msg", "configuration reloaded after debouncing")
 			})
 			reloadTimer.Stop()
 		}
@@ -67,7 +161,7 @@ func PathContentReloader(ctx context.Context, fileContent fileContent, logger lo
 				}
 				// We are watching the file's parent folder (more details on this is done can be found below), but are
 				// only interested in changed to the target file. Discard every other file as quickly as possible.
-				if event.Name != filePath {
+				if event.Name != f.filePath {
 					break
 				}
 				// We only react to files being written or created.
@@ -76,19 +170,19 @@ func PathContentReloader(ctx context.Context, fileContent fileContent, logger lo
 				if event.Op&fsnotify.Write == 0 && event.Op&fsnotify.Create == 0 {
 					break
 				}
-				level.Debug(logger).Log("msg", fmt.Sprintf("change detected for %s", filePath), "eventName", event.Name, "eventOp", event.Op)
+				level.Debug(f.logger).Log("msg", fmt.Sprintf("change detected for %s", f.filePath), "eventName", event.Name, "eventOp", event.Op)
 				if reloadTimer != nil {
-					reloadTimer.Reset(debounceTime)
+					reloadTimer.Reset(f.debounce)
 				}
 			case err := <-watcher.Errors:
-				level.Error(logger).Log("msg", "watcher error", "error", err)
+				level.Error(f.logger).Log("msg", "watcher error", "error", err)
 			}
 		}
 	}()
 	// We watch the file's parent folder and not the file itself to better handle DELETE and RENAME events. Check
 	// https://github.com/fsnotify/fsnotify/issues/214 for more details.
-	if err := watcher.Add(path.Dir(filePath)); err != nil {
-		return errors.Wrapf(err, "adding path %s to file watcher", filePath)
+	if err := watcher.Add(path.Dir(f.filePath)); err != nil {
+		return errors.Wrapf(err, "adding path %s to file watcher", f.filePath)
 	}
 	return nil
 }