in_tail: Add option file_cache_advise to reduce file cache usage

This commit will add option file_cache_advise to the tail plugin. It allows to set the posix_fadvise in POSIX_FADV_DONTNEED mode. This will reduce the usage of the kernel file cache. Signed-off-by: Richard Treu <[email protected]>
fluent · Jan 26, 2024 · df4e409 · df4e409
1 parent 71746b3
commit df4e409
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 1 deletion.
diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c
@@ -680,6 +680,13 @@ static struct flb_config_map config_map[] = {
      "Allows to skip empty lines."
     },
 
+#ifdef __linux__
+    {
+     FLB_CONFIG_MAP_BOOL, "file_cache_advise", "false",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, file_cache_advise),
+     "Use posix_fadvise for file access. Advise not to use kernel file cache."
+    },
+#endif
 #ifdef FLB_HAVE_INOTIFY
     {
      FLB_CONFIG_MAP_BOOL, "inotify_watcher", "true",

diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h
@@ -93,6 +93,9 @@ struct flb_tail_config {
     int   skip_long_lines;     /* skip long lines              */
     int   skip_empty_lines;    /* skip empty lines (off)       */
     int   exit_on_eof;         /* exit fluent-bit on EOF, test */
+#ifdef __linux__
+    int   file_cache_advise;   /* Use posix_fadvise for file access */
+#endif
 
     int progress_check_interval;      /* watcher interval             */
     int progress_check_interval_nsec; /* watcher interval             */

diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c
@@ -934,7 +934,13 @@ int flb_tail_file_append(char *path, struct stat *st, int mode,
     if (flb_tail_file_exists(st, ctx) == FLB_TRUE) {
         return -1;
     }
-
+
+    #ifdef __linux__
+    if (ctx->file_cache_advise) {
+        flb_plg_info(ctx->ins, "file will be read in POSIX_FADV_DONTNEED mode %s", path);
+    }
+    #endif
+
     fd = open(path, O_RDONLY);
     if (fd == -1) {
         flb_errno();
@@ -1399,6 +1405,14 @@ int flb_tail_file_chunk(struct flb_tail_file *file)
         capacity = (file->buf_size - file->buf_len) - 1;
     }
 
+    #ifdef __linux__
+    if (ctx->file_cache_advise) {
+        if (posix_fadvise(file->fd, 0, 0, POSIX_FADV_DONTNEED) == -1) {
+            flb_errno();
+            flb_plg_error(ctx->ins, "error during posix_fadvise");
+        }
+    }
+    #endif
     bytes = read(file->fd, file->buf_data + file->buf_len, capacity);
     if (bytes > 0) {
         /* we read some data, let the content processor take care of it */