fix excavate bug

bbot/core/event/base.py

@@ -1003,13 +1003,15 @@ def __init__(self, *args, **kwargs):
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -1559,6 +1561,8 @@ def __init__(self, *args, **kwargs):
                 self.add_tag("compressed")
                 self.add_tag(f"{compression}-archive")
                 self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
 
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
@@ -1639,23 +1643,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
+            event.discovery_context = context
         if internal == True:
-            data.internal = True
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.add_tags(tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)

bbot/modules/base.py

@@ -1628,7 +1628,7 @@ async def _worker(self):
                                 forward_event, forward_event_reason = forward_event
 
                         if forward_event is False:
-                            self.debug(f"Not forwarding {event} because {forward_event_reason}")
+                            self.debug(f"Not forwarding {event.type}:{event.data} because {forward_event_reason}")
                             continue
 
                     self.debug(f"Forwarding {event}")

bbot/test/bbot_fixtures.py

@@ -224,9 +224,9 @@ class bbot_events:
     return bbot_events
 
 
-@pytest.fixture(scope="session", autouse=True)
-def install_all_python_deps():
-    deps_pip = set()
-    for module in DEFAULT_PRESET.module_loader.preloaded().values():
-        deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+# @pytest.fixture(scope="session", autouse=True)
+# def install_all_python_deps():
+#     deps_pip = set()
+#     for module in DEFAULT_PRESET.module_loader.preloaded().values():
+#         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
+#     subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))

bbot/test/test_step_1/test_events.py

@@ -966,3 +966,35 @@ def test_event_magic():
     assert event.tags == {"folder"}
 
     zip_file.unlink()
+
+
+def test_event_hashing():
+    scan = Scanner("example.com")
+    url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
+    host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
+    host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
+    finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
+    finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
+    finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+    finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+
+    assert finding1.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "www.example.com",
+    }
+    assert finding2.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding3.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding1.id != finding2.id
+    assert finding2.id == finding3.id
+    assert finding1.data_id != finding2.data_id
+    assert finding2.data_id == finding3.data_id
+    assert finding1.data_hash != finding2.data_hash
+    assert finding2.data_hash == finding3.data_hash
+    assert hash(finding1) != hash(finding2)
+    assert hash(finding2) == hash(finding3)