in_tail: Add option file_cache_advise to reduce file cache usage

This commit will add option file_cache_advise to the tail plugin. It allows to set the posix_fadvise in POSIX_FADV_DONTNEED mode. This will reduce the usage of the kernel file cache. Feature is enabled by default. Signed-off-by: Richard Treu <[email protected]>
fluent · Jun 12, 2024 · 8542fac · 8542fac
1 parent b99fc4d
commit 8542fac
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 2 deletions.
diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c
@@ -696,7 +696,13 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct flb_tail_config, skip_empty_lines),
      "Allows to skip empty lines."
     },
-
+#ifdef __linux__
+    {
+     FLB_CONFIG_MAP_BOOL, "file_cache_advise", "true",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, file_cache_advise),
+     "Use posix_fadvise for file access. Advise not to use kernel file cache."
+    },
+#endif
 #ifdef FLB_HAVE_INOTIFY
     {
      FLB_CONFIG_MAP_BOOL, "inotify_watcher", "true",

diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h
@@ -93,6 +93,9 @@ struct flb_tail_config {
     int   skip_long_lines;     /* skip long lines              */
     int   skip_empty_lines;    /* skip empty lines (off)       */
     int   exit_on_eof;         /* exit fluent-bit on EOF, test */
+#ifdef __linux__
+    int   file_cache_advise;   /* Use posix_fadvise for file access */
+#endif
 
     int progress_check_interval;      /* watcher interval             */
     int progress_check_interval_nsec; /* watcher interval             */

diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c
@@ -940,7 +940,13 @@ int flb_tail_file_append(char *path, struct stat *st, int mode,
     if (flb_tail_file_exists(st, ctx) == FLB_TRUE) {
         return -1;
     }
-
+
+    #ifdef __linux__
+    if (ctx->file_cache_advise) {
+        flb_plg_debug(ctx->ins, "file will be read in POSIX_FADV_DONTNEED mode %s", path);
+    }
+    #endif
+
     fd = open(path, O_RDONLY);
     if (fd == -1) {
         flb_errno();
@@ -1429,6 +1435,15 @@ int flb_tail_file_chunk(struct flb_tail_file *file)
         file_buffer_capacity = (file->buf_size - file->buf_len) - 1;
     }
 
+    #ifdef __linux__
+    if (ctx->file_cache_advise) {
+        if (posix_fadvise(file->fd, 0, 0, POSIX_FADV_DONTNEED) == -1) {
+            flb_errno();
+            flb_plg_error(ctx->ins, "error during posix_fadvise");
+        }
+    }
+    #endif
+
     read_size = file_buffer_capacity;
 
     if (file->decompression_context != NULL) {