Merge pull request #1049 from blacklanternsecurity/siem-friendly-json

JSON module: option for SIEM-friendly output

bbot/modules/output/json.py

@@ -7,16 +7,24 @@
 class JSON(BaseOutputModule):
     watched_events = ["*"]
     meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"}
-    options = {"output_file": "", "console": False}
-    options_desc = {"output_file": "Output to file", "console": "Output to console"}
+    options = {"output_file": "", "console": False, "siem_friendly": False}
+    options_desc = {
+        "output_file": "Output to file",
+        "console": "Output to console",
+        "siem_friendly": "Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc.",
+    }
     _preserve_graph = True
 
     async def setup(self):
         self._prep_output_dir("output.ndjson")
+        self.siem_friendly = self.config.get("siem_friendly", False)
         return True
 
     async def handle_event(self, event):
-        event_str = json.dumps(dict(event))
+        event_json = dict(event)
+        if self.siem_friendly:
+            event_json["data"] = {event.type: event_json.pop("data", "")}
+        event_str = json.dumps(event_json)
         if self.file is not None:
             self.file.write(event_str + "\n")
             self.file.flush()

bbot/test/test_step_2/module_tests/test_module_json.py

@@ -12,3 +12,18 @@ def check(self, module_test, events):
         e = event_from_json(json.loads(lines[0]))
         assert e.type == "SCAN"
         assert e.data == f"{module_test.scan.name} ({module_test.scan.id})"
+
+
+class TestJSONSIEMFriendly(ModuleTestBase):
+    modules_overrides = ["json"]
+    config_overrides = {"output_modules": {"json": {"siem_friendly": True}}}
+
+    def check(self, module_test, events):
+        txt_file = module_test.scan.home / "output.ndjson"
+        lines = list(module_test.scan.helpers.read_file(txt_file))
+        passed = False
+        for line in lines:
+            e = json.loads(line)
+            if e["data"] == {"DNS_NAME": "blacklanternsecurity.com"}:
+                passed = True
+        assert passed

docs/scanning/tips_and_tricks.md

@@ -53,6 +53,24 @@ You can also pair the web spider with subdomain enumeration:
 bbot -t evilcorp.com -f subdomain-enum -c spider.yml
 ```
 
+### Ingesting BBOT Data Into SIEM (Elastic, Splunk)
+
+If your goal is to feed BBOT data into a SIEM such as Elastic, be sure to enable this option when scanning:
+
+```bash
+bbot -t evilcorp.com -c output_modules.json.siem_friendly=true
+```
+
+This nests the event's `.data` beneath its event type like so:
+```json
+{
+  "type": "DNS_NAME",
+  "data": {
+    "DNS_NAME": "blacklanternsecurity.com"
+  }
+}
+```
+
 ### Custom HTTP Proxy
 
 Web pentesters may appreciate BBOT's ability to quickly populate Burp Suite site maps for all subdomains in a target. If your scan includes gowitness, this will capture the traffic as if you manually visited each website in your browser -- including auxiliary web resources and javascript API calls. To accomplish this, set the `http_proxy` config option like so: