out_splunk: allow user to send raw message by setting Splunk_Send_Raw…

… On (#610) This patch adds a new toggle for the Splunk HTTP Event Collector output plugin to allow users to specify index, sourcetype and other top level fields when sending data to splunk. The current behaviour sends data to the main index by default. When enabling the feature, the user must take care to put all log details in the event field, and only specify fields known to splunk in the top level event. Example: Splunk_Send_Raw Off {"time": .., "event": {"k1": "foo", "k2": "bar", "index": "applogs"}} Splunk_Send_Raw On {"time": .., "k1": "foo", "k2": "bar", "index": "applogs"} For up to date information about the valid keys in the top level object, refer to the splunk documentation: http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC The patch has been tested using Splunk 6.6.1 Signed-off-by: Carl Henrik Lunde <[email protected]>
fluent · Jun 15, 2018 · d3bf2c2 · d3bf2c2
1 parent 51739bb
commit d3bf2c2
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 7 deletions.
diff --git a/plugins/out_splunk/splunk.c b/plugins/out_splunk/splunk.c
@@ -87,7 +87,11 @@ int splunk_format(void *in_buf, size_t in_bytes,
         map = root.via.array.ptr[1];
         map_size = map.via.map.size;
 
-        msgpack_pack_map(&mp_pck, 2);
+        if (ctx->splunk_send_raw) {
+            msgpack_pack_map(&mp_pck, 1 + map_size /* time + all k/v */);
+        } else {
+            msgpack_pack_map(&mp_pck, 2 /* time + event */);
+        }
 
         /* Append the time key */
         msgpack_pack_str(&mp_pck, sizeof(FLB_SPLUNK_DEFAULT_TIME) -1);
@@ -96,12 +100,14 @@ int splunk_format(void *in_buf, size_t in_bytes,
                               sizeof(FLB_SPLUNK_DEFAULT_TIME) - 1);
         msgpack_pack_double(&mp_pck, t);
 
-        /* Append event key */
-        msgpack_pack_str(&mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT) -1);
-        msgpack_pack_str_body(&mp_pck,
-                              FLB_SPLUNK_DEFAULT_EVENT,
-                              sizeof(FLB_SPLUNK_DEFAULT_EVENT) - 1);
-        msgpack_pack_map(&mp_pck, map_size);
+        if (!ctx->splunk_send_raw) {
+            /* Add k/v pairs under the key 'event' instead of to the top level object */
+            msgpack_pack_str(&mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT) -1);
+            msgpack_pack_str_body(&mp_pck,
+                                  FLB_SPLUNK_DEFAULT_EVENT,
+                                  sizeof(FLB_SPLUNK_DEFAULT_EVENT) - 1);
+            msgpack_pack_map(&mp_pck, map_size);
+        }
 
         /* Append k/v */
         for (i = 0; i < map_size; i++) {

diff --git a/plugins/out_splunk/splunk.h b/plugins/out_splunk/splunk.h
@@ -38,6 +38,9 @@ struct flb_splunk {
     /* Token Auth */
     flb_sds_t auth_header;
 
+    /* Send fields directly or pack data into "event" object */
+    int splunk_send_raw;
+
     /* Upstream connection to the backend server */
     struct flb_upstream *u;
 };

diff --git a/plugins/out_splunk/splunk_conf.c b/plugins/out_splunk/splunk_conf.c
@@ -110,6 +110,15 @@ struct flb_splunk *flb_splunk_conf_create(struct flb_output_instance *ins,
         }
     }
 
+    /* Event format, send all fields or pack into event map */
+    tmp = flb_output_get_property("splunk_send_raw", ins);
+    if (tmp) {
+        ctx->splunk_send_raw = flb_utils_bool(tmp);
+    }
+    else {
+        ctx->splunk_send_raw = FLB_FALSE;
+    }
+
     return ctx;
 }