Merge pull request #1568 from blacklanternsecurity/discovery-ids

Include event IDs in discovery path

bbot/core/event/base.py

@@ -354,7 +354,7 @@ def discovery_path(self):
         This event's full discovery context, including those of all its parents
         """
         full_event_chain = list(reversed(self.get_parents())) + [self]
-        return [e.discovery_context for e in full_event_chain if e.type != "SCAN"]
+        return [[e.id, e.discovery_context] for e in full_event_chain if e.type != "SCAN"]
 
     @property
     def words(self):

bbot/modules/output/csv.py

@@ -54,6 +54,8 @@ def writerow(self, row):
 
     async def handle_event(self, event):
         # ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"]
+        discovery_path = getattr(event, "discovery_path", [])
+        discovery_path = [e[-1] for e in discovery_path]
         self.writerow(
             {
                 "Event type": getattr(event, "type", ""),
@@ -64,7 +66,7 @@ async def handle_event(self, event):
                 "Source Module": str(getattr(event, "module_sequence", "")),
                 "Scope Distance": str(getattr(event, "scope_distance", "")),
                 "Event Tags": ",".join(sorted(list(getattr(event, "tags", [])))),
-                "Discovery Path": " --> ".join(getattr(event, "discovery_path", [])),
+                "Discovery Path": " --> ".join(discovery_path),
             }
         )
 

bbot/test/test_step_1/test_events.py

@@ -408,7 +408,7 @@ async def test_events(events, helpers):
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     assert db_event.discovery_context == "test context"
-    assert db_event.discovery_path == ["test context"]
+    assert db_event.discovery_path == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]]
     timestamp = db_event.timestamp.isoformat()
     json_event = db_event.json()
     assert json_event["scope_distance"] == 1
@@ -417,15 +417,17 @@ async def test_events(events, helpers):
     assert json_event["host"] == "evilcorp.com"
     assert json_event["timestamp"] == timestamp
     assert json_event["discovery_context"] == "test context"
-    assert json_event["discovery_path"] == ["test context"]
+    assert json_event["discovery_path"] == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]]
     reconstituted_event = event_from_json(json_event)
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.isoformat() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
     assert reconstituted_event.type == "OPEN_TCP_PORT"
     assert reconstituted_event.host == "evilcorp.com"
     assert reconstituted_event.discovery_context == "test context"
-    assert reconstituted_event.discovery_path == ["test context"]
+    assert reconstituted_event.discovery_path == [
+        ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]
+    ]
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
@@ -596,7 +598,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "evilcorp.com"
             and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"
-            and e.discovery_path == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
+            and [_[-1] for _ in e.discovery_path] == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
         ]
     )
     assert 1 == len(
@@ -606,7 +608,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "one.evilcorp.com"
             and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com"
-            and e.discovery_path
+            and [_[-1] for _ in e.discovery_path]
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -621,7 +623,7 @@ async def handle_event(self, event):
             and e.data == "two.evilcorp.com"
             and e.discovery_context
             == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com"
-            and e.discovery_path
+            and [_[-1] for _ in e.discovery_path]
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -636,7 +638,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "three.evilcorp.com"
             and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com"
-            and e.discovery_path
+            and [_[-1] for _ in e.discovery_path]
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -658,11 +660,11 @@ async def handle_event(self, event):
         if e.type == "DNS_NAME"
         and e.data == "four.evilcorp.com"
         and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com"
-        and e.discovery_path == final_path
+        and [_[-1] for _ in e.discovery_path] == final_path
     ]
     assert 1 == len(final_event)
     j = final_event[0].json()
-    assert j["discovery_path"] == final_path
+    assert [_[-1] for _ in j["discovery_path"]] == final_path
 
     await scan._cleanup()
 
@@ -675,6 +677,6 @@ async def handle_event(self, event):
     events = [e async for e in scan.async_start()]
     blsops_event = [e for e in events if e.type == "DNS_NAME" and e.data == "blsops.com"]
     assert len(blsops_event) == 1
-    assert blsops_event[0].discovery_path[1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com"
+    assert blsops_event[0].discovery_path[1][-1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com"
 
     await scan._cleanup()

bbot/test/test_step_2/module_tests/test_module_json.py

@@ -26,7 +26,7 @@ def check(self, module_test, events):
         assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"]
         assert dns_json["data"] == dns_data
         assert dns_json["discovery_context"] == context_data
-        assert dns_json["discovery_path"] == [context_data]
+        assert dns_json["discovery_path"] == [["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data]]
 
         # event objects reconstructed from json
         scan_reconstructed = event_from_json(scan_json)
@@ -37,7 +37,9 @@ def check(self, module_test, events):
         assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"]
         assert dns_reconstructed.data == dns_data
         assert dns_reconstructed.discovery_context == context_data
-        assert dns_reconstructed.discovery_path == [context_data]
+        assert dns_reconstructed.discovery_path == [
+            ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data]
+        ]
 
 
 class TestJSONSIEMFriendly(ModuleTestBase):