add siem_friendly option to http output module

bbot/core/event/base.py

@@ -565,7 +565,7 @@ def __contains__(self, other):
             return host_in_host(other.host, self.host)
         return False
 
-    def json(self, mode="json"):
+    def json(self, mode="json", siem_friendly=False):
         """
         Serializes the event object to a JSON-compatible dictionary.
 
@@ -574,6 +574,7 @@ def json(self, mode="json"):
 
         Parameters:
             mode (str): Specifies the data serialization mode. Default is "json". Other options include "graph", "human", and "id".
+            siem_friendly (bool): Whether to format the JSON in a way that's friendly to SIEM ingestion by Elastic, Splunk, etc. This ensures the value of "data" is always the same type (a dictionary).
 
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
@@ -585,9 +586,13 @@ def json(self, mode="json"):
                 j.update({i: v})
         data_attr = getattr(self, f"data_{mode}", None)
         if data_attr is not None:
-            j["data"] = data_attr
+            data = data_attr
         else:
-            j["data"] = smart_decode(self.data)
+            data = smart_decode(self.data)
+        if siem_friendly:
+            j["data"] = {self.type: data}
+        else:
+            j["data"] = data
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
             j["web_spider_distance"] = web_spider_distance
@@ -1312,7 +1317,7 @@ def make_event(
         )
 
 
-def event_from_json(j):
+def event_from_json(j, siem_friendly=False):
     """
     Creates an event object from a JSON dictionary.
 
@@ -1335,14 +1340,19 @@ def event_from_json(j):
         if required keys are missing. Make sure to validate the JSON input beforehand.
     """
     try:
+        event_type = j["type"]
         kwargs = {
-            "data": j["data"],
-            "event_type": j["type"],
+            "event_type": event_type,
             "scans": j.get("scans", []),
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 5),
             "dummy": True,
         }
+        if siem_friendly:
+            data = j["data"][event_type]
+        else:
+            data = j["data"]
+        kwargs["data"] = data
         event = make_event(**kwargs)
 
         resolved_hosts = j.get("resolved_hosts", [])

bbot/modules/output/http.py

@@ -13,6 +13,7 @@ class HTTP(BaseOutputModule):
         "username": "",
         "password": "",
         "timeout": 10,
+        "siem_friendly": False,
     }
     options_desc = {
         "url": "Web URL",
@@ -21,12 +22,14 @@ class HTTP(BaseOutputModule):
         "username": "Username (basic auth)",
         "password": "Password (basic auth)",
         "timeout": "HTTP timeout",
+        "siem_friendly": "Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc.",
     }
 
     async def setup(self):
         self.url = self.config.get("url", "")
         self.method = self.config.get("method", "POST")
         self.timeout = self.config.get("timeout", 10)
+        self.siem_friendly = self.config.get("siem_friendly", False)
         self.headers = {}
         bearer = self.config.get("bearer", "")
         if bearer:
@@ -52,7 +55,7 @@ async def handle_event(self, event):
                     method=self.method,
                     auth=self.auth,
                     headers=self.headers,
-                    json=dict(event),
+                    json=event.json(siem_friendly=self.siem_friendly),
                     raise_error=True,
                 )
                 break

bbot/modules/output/json.py

@@ -21,9 +21,7 @@ async def setup(self):
         return True
 
     async def handle_event(self, event):
-        event_json = dict(event)
-        if self.siem_friendly:
-            event_json["data"] = {event.type: event_json.pop("data", "")}
+        event_json = event.json(siem_friendly=self.siem_friendly)
         event_str = json.dumps(event_json)
         if self.file is not None:
             self.file.write(event_str + "\n")

bbot/test/test_step_1/test_events.py

@@ -385,6 +385,19 @@ async def test_events(events, scan, helpers, bbot_config):
     assert reconstituted_event.type == "DNS_NAME"
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
 
+    # SIEM-friendly serialize/deserialize
+    json_event_siemfriendly = db_event.json(siem_friendly=True)
+    assert json_event_siemfriendly["scope_distance"] == 1
+    assert json_event_siemfriendly["data"] == {"DNS_NAME": "evilcorp.com"}
+    assert json_event_siemfriendly["type"] == "DNS_NAME"
+    assert json_event_siemfriendly["timestamp"] == timestamp
+    reconstituted_event2 = event_from_json(json_event_siemfriendly, siem_friendly=True)
+    assert reconstituted_event2.scope_distance == 1
+    assert reconstituted_event2.timestamp.timestamp() == timestamp
+    assert reconstituted_event2.data == "evilcorp.com"
+    assert reconstituted_event2.type == "DNS_NAME"
+    assert "127.0.0.1" in reconstituted_event2.resolved_hosts
+
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)
     assert http_response.source_id == scan.root_event.id
     assert http_response.data["input"] == "http://example.com:80"

bbot/test/test_step_2/module_tests/test_module_http.py

@@ -1,3 +1,6 @@
+import json
+import httpx
+
 from .base import ModuleTestBase
 
 
@@ -15,10 +18,46 @@ class TestHTTP(ModuleTestBase):
         }
     }
 
+    def verify_data(self, j):
+        return j["data"] == "blacklanternsecurity.com" and j["type"] == "DNS_NAME"
+
     async def setup_after_prep(self, module_test):
+        self.got_event = False
+        self.headers_correct = False
+        self.method_correct = False
+        self.url_correct = False
+
+        async def custom_callback(request):
+            j = json.loads(request.content)
+            if request.url == self.downstream_url:
+                self.url_correct = True
+            if request.method == "PUT":
+                self.method_correct = True
+            if "Authorization" in request.headers:
+                self.headers_correct = True
+            if self.verify_data(j):
+                self.got_event = True
+            return httpx.Response(
+                status_code=200,
+            )
+
+        module_test.httpx_mock.add_callback(custom_callback)
+        module_test.httpx_mock.add_callback(custom_callback)
         module_test.httpx_mock.add_response(
             method="PUT", headers={"Authorization": "bearer auth_token"}, url=self.downstream_url
         )
 
     def check(self, module_test, events):
-        pass
+        assert self.got_event == True
+        assert self.headers_correct == True
+        assert self.method_correct == True
+        assert self.url_correct == True
+
+
+class TestHTTPSIEMFriendly(TestHTTP):
+    modules_overrides = ["http"]
+    config_overrides = TestHTTP.config_overrides
+    config_overrides["output_modules"]["http"]["siem_friendly"] = True
+
+    def verify_data(self, j):
+        return j["data"] == {"DNS_NAME": "blacklanternsecurity.com"} and j["type"] == "DNS_NAME"