Merge branch 'dev' into lightfuzz

bbot/core/helpers/names_generator.py

@@ -216,6 +216,7 @@
     "sneaky",
     "soft",
     "sophisticated",
+    "spicy",
     "spiteful",
     "squishy",
     "steamy",
@@ -456,6 +457,7 @@
     "hermione",
     "homer",
     "howard",
+    "hunter",
     "irene",
     "isaac",
     "isabella",

bbot/modules/trufflehog.py

@@ -14,7 +14,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.82.6",
+        "version": "3.82.7",
         "config": "",
         "only_verified": True,
         "concurrency": 8,
@@ -31,7 +31,7 @@ class trufflehog(BaseModule):
         {
             "name": "Download trufflehog",
             "unarchive": {
-                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
+                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
                 "include": "trufflehog",
                 "dest": "#{BBOT_TOOLS}",
                 "remote_src": True,

bbot/scanner/scanner.py

@@ -116,6 +116,7 @@ def __init__(
             **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`).
         """
         self._root_event = None
+        self._finish_event = None
         self.start_time = None
         self.end_time = None
         self.duration = None
@@ -383,8 +384,8 @@ async def async_start(self):
                     new_activity = await self.finish()
                     if not new_activity:
                         self._success = True
-                        await self._mark_finished()
-                        yield self.root_event
+                        scan_finish_event = await self._mark_finished()
+                        yield scan_finish_event
                         break
 
                 await asyncio.sleep(0.1)
@@ -440,8 +441,7 @@ async def _mark_finished(self):
 
         status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}"
 
-        scan_finish_event = self.make_root_event(status_message)
-        scan_finish_event.data["status"] = status
+        scan_finish_event = self.finish_event(status_message, status)
 
         # queue final scan event with output modules
         output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"]
@@ -457,6 +457,7 @@ async def _mark_finished(self):
 
         self.status = status
         log_fn(status_message)
+        return scan_finish_event
 
     def _start_modules(self):
         self.verbose(f"Starting module worker loops")
@@ -996,6 +997,14 @@ def root_event(self):
         self._root_event.data["status"] = self.status
         return self._root_event
 
+    def finish_event(self, context=None, status=None):
+        if self._finish_event is None:
+            if context is None or status is None:
+                raise ValueError("Must specify context and status")
+            self._finish_event = self.make_root_event(context)
+            self._finish_event.data["status"] = status
+        return self._finish_event
+
     def make_root_event(self, context):
         root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True, context=context)
         root_event._id = self.id

bbot/test/test_step_1/test_helpers.py

@@ -49,6 +49,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.url_depth("http://evilcorp.com/") == 0
     assert helpers.url_depth("http://evilcorp.com") == 0
 
+    assert helpers.parent_url("http://evilcorp.com/subdir1/subdir2?foo=bar") == "http://evilcorp.com/subdir1"
+
     ### MISC ###
     assert helpers.is_domain("evilcorp.co.uk")
     assert not helpers.is_domain("www.evilcorp.co.uk")

bbot/test/test_step_1/test_scan.py

@@ -118,3 +118,28 @@ async def test_speed_counter():
         await asyncio.sleep(0.2)
     # only 5 should show
     assert 4 <= counter.speed <= 5
+
+
+@pytest.mark.asyncio
+async def test_python_output_matches_json(bbot_scanner):
+    import json
+
+    scan = bbot_scanner(
+        "blacklanternsecurity.com",
+        config={"speculate": True, "dns": {"minimal": False}, "scope": {"report_distance": 10}},
+    )
+    await scan.helpers.dns._mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.1"]}})
+    events = [e.json() async for e in scan.async_start()]
+    output_json = scan.home / "output.json"
+    json_events = []
+    for line in open(output_json):
+        json_events.append(json.loads(line))
+
+    assert len(events) == 5
+    scan_events = [e for e in events if e["type"] == "SCAN"]
+    assert len(scan_events) == 2
+    assert all([isinstance(e["data"]["status"], str) for e in scan_events])
+    assert len([e for e in events if e["type"] == "DNS_NAME"]) == 1
+    assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1
+    assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1
+    assert events == json_events

pyproject.toml

@@ -63,7 +63,7 @@ poetry-dynamic-versioning = ">=0.21.4,<1.5.0"
 urllib3 = "^2.0.2"
 werkzeug = ">=2.3.4,<4.0.0"
 pytest-env = ">=0.8.2,<1.2.0"
-pre-commit = "^3.4.0"
+pre-commit = ">=3.4,<5.0"
 black = "^24.1.1"
 pytest-cov = "^5.0.0"
 pytest-rerunfailures = "^14.0"