diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 24606f890..0e63b6291 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -565,7 +565,7 @@ def __contains__(self, other): return host_in_host(other.host, self.host) return False - def json(self, mode="json"): + def json(self, mode="json", siem_friendly=False): """ Serializes the event object to a JSON-compatible dictionary. @@ -574,6 +574,7 @@ def json(self, mode="json"): Parameters: mode (str): Specifies the data serialization mode. Default is "json". Other options include "graph", "human", and "id". + siem_friendly (bool): Whether to format the JSON in a way that's friendly to SIEM ingestion by Elastic, Splunk, etc. This ensures the value of "data" is always the same type (a dictionary). Returns: dict: JSON-serializable dictionary representation of the event object. @@ -585,9 +586,13 @@ def json(self, mode="json"): j.update({i: v}) data_attr = getattr(self, f"data_{mode}", None) if data_attr is not None: - j["data"] = data_attr + data = data_attr else: - j["data"] = smart_decode(self.data) + data = smart_decode(self.data) + if siem_friendly: + j["data"] = {self.type: data} + else: + j["data"] = data web_spider_distance = getattr(self, "web_spider_distance", None) if web_spider_distance is not None: j["web_spider_distance"] = web_spider_distance @@ -1312,7 +1317,7 @@ def make_event( ) -def event_from_json(j): +def event_from_json(j, siem_friendly=False): """ Creates an event object from a JSON dictionary. @@ -1335,14 +1340,19 @@ def event_from_json(j): if required keys are missing. Make sure to validate the JSON input beforehand. """ try: + event_type = j["type"] kwargs = { - "data": j["data"], - "event_type": j["type"], + "event_type": event_type, "scans": j.get("scans", []), "tags": j.get("tags", []), "confidence": j.get("confidence", 5), "dummy": True, } + if siem_friendly: + data = j["data"][event_type] + else: + data = j["data"] + kwargs["data"] = data event = make_event(**kwargs) resolved_hosts = j.get("resolved_hosts", []) diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 10ca1c8df..014610736 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -13,6 +13,7 @@ class HTTP(BaseOutputModule): "username": "", "password": "", "timeout": 10, + "siem_friendly": False, } options_desc = { "url": "Web URL", @@ -21,12 +22,14 @@ class HTTP(BaseOutputModule): "username": "Username (basic auth)", "password": "Password (basic auth)", "timeout": "HTTP timeout", + "siem_friendly": "Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc.", } async def setup(self): self.url = self.config.get("url", "") self.method = self.config.get("method", "POST") self.timeout = self.config.get("timeout", 10) + self.siem_friendly = self.config.get("siem_friendly", False) self.headers = {} bearer = self.config.get("bearer", "") if bearer: @@ -52,7 +55,7 @@ async def handle_event(self, event): method=self.method, auth=self.auth, headers=self.headers, - json=dict(event), + json=event.json(siem_friendly=self.siem_friendly), raise_error=True, ) break diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index a380ac9a1..bf8517db9 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -21,9 +21,7 @@ async def setup(self): return True async def handle_event(self, event): - event_json = dict(event) - if self.siem_friendly: - event_json["data"] = {event.type: event_json.pop("data", "")} + event_json = event.json(siem_friendly=self.siem_friendly) event_str = json.dumps(event_json) if self.file is not None: self.file.write(event_str + "\n") diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index ccbf17c1f..cadde29ad 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -385,6 +385,19 @@ async def test_events(events, scan, helpers, bbot_config): assert reconstituted_event.type == "DNS_NAME" assert "127.0.0.1" in reconstituted_event.resolved_hosts + # SIEM-friendly serialize/deserialize + json_event_siemfriendly = db_event.json(siem_friendly=True) + assert json_event_siemfriendly["scope_distance"] == 1 + assert json_event_siemfriendly["data"] == {"DNS_NAME": "evilcorp.com"} + assert json_event_siemfriendly["type"] == "DNS_NAME" + assert json_event_siemfriendly["timestamp"] == timestamp + reconstituted_event2 = event_from_json(json_event_siemfriendly, siem_friendly=True) + assert reconstituted_event2.scope_distance == 1 + assert reconstituted_event2.timestamp.timestamp() == timestamp + assert reconstituted_event2.data == "evilcorp.com" + assert reconstituted_event2.type == "DNS_NAME" + assert "127.0.0.1" in reconstituted_event2.resolved_hosts + http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event) assert http_response.source_id == scan.root_event.id assert http_response.data["input"] == "http://example.com:80" diff --git a/bbot/test/test_step_2/module_tests/test_module_http.py b/bbot/test/test_step_2/module_tests/test_module_http.py index 3b4e819b9..44a969370 100644 --- a/bbot/test/test_step_2/module_tests/test_module_http.py +++ b/bbot/test/test_step_2/module_tests/test_module_http.py @@ -1,3 +1,6 @@ +import json +import httpx + from .base import ModuleTestBase @@ -15,10 +18,46 @@ class TestHTTP(ModuleTestBase): } } + def verify_data(self, j): + return j["data"] == "blacklanternsecurity.com" and j["type"] == "DNS_NAME" + async def setup_after_prep(self, module_test): + self.got_event = False + self.headers_correct = False + self.method_correct = False + self.url_correct = False + + async def custom_callback(request): + j = json.loads(request.content) + if request.url == self.downstream_url: + self.url_correct = True + if request.method == "PUT": + self.method_correct = True + if "Authorization" in request.headers: + self.headers_correct = True + if self.verify_data(j): + self.got_event = True + return httpx.Response( + status_code=200, + ) + + module_test.httpx_mock.add_callback(custom_callback) + module_test.httpx_mock.add_callback(custom_callback) module_test.httpx_mock.add_response( method="PUT", headers={"Authorization": "bearer auth_token"}, url=self.downstream_url ) def check(self, module_test, events): - pass + assert self.got_event == True + assert self.headers_correct == True + assert self.method_correct == True + assert self.url_correct == True + + +class TestHTTPSIEMFriendly(TestHTTP): + modules_overrides = ["http"] + config_overrides = TestHTTP.config_overrides + config_overrides["output_modules"]["http"]["siem_friendly"] = True + + def verify_data(self, j): + return j["data"] == {"DNS_NAME": "blacklanternsecurity.com"} and j["type"] == "DNS_NAME"