From 74b0f5c51db547874dff470c82500ef4d7279584 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Jul 2024 07:39:59 -0400 Subject: [PATCH 1/2] include event ids in discovery context --- bbot/core/event/base.py | 2 +- bbot/modules/output/csv.py | 4 +++- bbot/test/test_step_1/test_events.py | 22 +++++++++---------- .../module_tests/test_module_json.py | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index c23769aad..0e3be6123 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -354,7 +354,7 @@ def discovery_path(self): This event's full discovery context, including those of all its parents """ full_event_chain = list(reversed(self.get_parents())) + [self] - return [e.discovery_context for e in full_event_chain if e.type != "SCAN"] + return [[e.id, e.discovery_context] for e in full_event_chain if e.type != "SCAN"] @property def words(self): diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index d48e9cd1d..3b214918b 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -54,6 +54,8 @@ def writerow(self, row): async def handle_event(self, event): # ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"] + discovery_path = getattr(event, "discovery_path", []) + discovery_path = [e[-1] for e in discovery_path] self.writerow( { "Event type": getattr(event, "type", ""), @@ -64,7 +66,7 @@ async def handle_event(self, event): "Source Module": str(getattr(event, "module_sequence", "")), "Scope Distance": str(getattr(event, "scope_distance", "")), "Event Tags": ",".join(sorted(list(getattr(event, "tags", [])))), - "Discovery Path": " --> ".join(getattr(event, "discovery_path", [])), + "Discovery Path": " --> ".join(discovery_path), } ) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 8b7503d13..cd9f62ce3 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -408,8 +408,8 @@ async def test_events(events, helpers): db_event._resolved_hosts = {"127.0.0.1"} db_event.scope_distance = 1 assert db_event.discovery_context == "test context" - assert db_event.discovery_path == ["test context"] - timestamp = db_event.timestamp.timestamp() + assert db_event.discovery_path == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] + timestamp = db_event.timestamp.isoformat() json_event = db_event.json() assert json_event["scope_distance"] == 1 assert json_event["data"] == "evilcorp.com:80" @@ -417,7 +417,7 @@ async def test_events(events, helpers): assert json_event["host"] == "evilcorp.com" assert json_event["timestamp"] == timestamp assert json_event["discovery_context"] == "test context" - assert json_event["discovery_path"] == ["test context"] + assert json_event["discovery_path"] == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] reconstituted_event = event_from_json(json_event) assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.timestamp() == timestamp @@ -425,7 +425,7 @@ async def test_events(events, helpers): assert reconstituted_event.type == "OPEN_TCP_PORT" assert reconstituted_event.host == "evilcorp.com" assert reconstituted_event.discovery_context == "test context" - assert reconstituted_event.discovery_path == ["test context"] + assert reconstituted_event.discovery_path == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] assert "127.0.0.1" in reconstituted_event.resolved_hosts hostless_event = scan.make_event("asdf", "ASDF", dummy=True) hostless_event_json = hostless_event.json() @@ -594,7 +594,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "evilcorp.com" and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com" - and e.discovery_path == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"] + and [_[-1] for _ in e.discovery_path] == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"] ] ) assert 1 == len( @@ -604,7 +604,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "one.evilcorp.com" and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com" - and e.discovery_path + and [_[-1] for _ in e.discovery_path] == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -619,7 +619,7 @@ async def handle_event(self, event): and e.data == "two.evilcorp.com" and e.discovery_context == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com" - and e.discovery_path + and [_[-1] for _ in e.discovery_path] == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -634,7 +634,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "three.evilcorp.com" and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com" - and e.discovery_path + and [_[-1] for _ in e.discovery_path] == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -656,11 +656,11 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "four.evilcorp.com" and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com" - and e.discovery_path == final_path + and [_[-1] for _ in e.discovery_path] == final_path ] assert 1 == len(final_event) j = final_event[0].json() - assert j["discovery_path"] == final_path + assert [_[-1] for _ in j["discovery_path"]] == final_path # test to make sure this doesn't come back # https://github.com/blacklanternsecurity/bbot/issues/1498 @@ -671,4 +671,4 @@ async def handle_event(self, event): events = [e async for e in scan.async_start()] blsops_event = [e for e in events if e.type == "DNS_NAME" and e.data == "blsops.com"] assert len(blsops_event) == 1 - assert blsops_event[0].discovery_path[1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com" + assert blsops_event[0].discovery_path[1][-1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com" diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 6a5215f6e..77bc7a3fb 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -26,7 +26,7 @@ def check(self, module_test, events): assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["discovery_context"] == context_data - assert dns_json["discovery_path"] == [context_data] + assert dns_json["discovery_path"] == [['DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc', context_data]] # event objects reconstructed from json scan_reconstructed = event_from_json(scan_json) @@ -37,7 +37,7 @@ def check(self, module_test, events): assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data assert dns_reconstructed.discovery_context == context_data - assert dns_reconstructed.discovery_path == [context_data] + assert dns_reconstructed.discovery_path == [['DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc', context_data]] class TestJSONSIEMFriendly(ModuleTestBase): From dfc37bcd160e1e879dc8ded97b556b812d3b471c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Jul 2024 07:43:25 -0400 Subject: [PATCH 2/2] blacked --- bbot/test/test_step_1/test_events.py | 8 +++++--- bbot/test/test_step_2/module_tests/test_module_json.py | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index cd9f62ce3..a5127fcff 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -408,7 +408,7 @@ async def test_events(events, helpers): db_event._resolved_hosts = {"127.0.0.1"} db_event.scope_distance = 1 assert db_event.discovery_context == "test context" - assert db_event.discovery_path == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] + assert db_event.discovery_path == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]] timestamp = db_event.timestamp.isoformat() json_event = db_event.json() assert json_event["scope_distance"] == 1 @@ -417,7 +417,7 @@ async def test_events(events, helpers): assert json_event["host"] == "evilcorp.com" assert json_event["timestamp"] == timestamp assert json_event["discovery_context"] == "test context" - assert json_event["discovery_path"] == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] + assert json_event["discovery_path"] == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]] reconstituted_event = event_from_json(json_event) assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.timestamp() == timestamp @@ -425,7 +425,9 @@ async def test_events(events, helpers): assert reconstituted_event.type == "OPEN_TCP_PORT" assert reconstituted_event.host == "evilcorp.com" assert reconstituted_event.discovery_context == "test context" - assert reconstituted_event.discovery_path == [['OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac', 'test context']] + assert reconstituted_event.discovery_path == [ + ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"] + ] assert "127.0.0.1" in reconstituted_event.resolved_hosts hostless_event = scan.make_event("asdf", "ASDF", dummy=True) hostless_event_json = hostless_event.json() diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 77bc7a3fb..6ccf4d847 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -26,7 +26,7 @@ def check(self, module_test, events): assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["discovery_context"] == context_data - assert dns_json["discovery_path"] == [['DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc', context_data]] + assert dns_json["discovery_path"] == [["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data]] # event objects reconstructed from json scan_reconstructed = event_from_json(scan_json) @@ -37,7 +37,9 @@ def check(self, module_test, events): assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data assert dns_reconstructed.discovery_context == context_data - assert dns_reconstructed.discovery_path == [['DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc', context_data]] + assert dns_reconstructed.discovery_path == [ + ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data] + ] class TestJSONSIEMFriendly(ModuleTestBase):