Merge pull request #2105 from blacklanternsecurity/misc-bugfixes

More Misc Bugfixes

bbot/core/event/base.py

@@ -590,7 +590,7 @@ def parent(self, parent):
                         if t in ("spider-danger", "spider-max"):
                             self.add_tag(t)
         elif not self._dummy:
-            log.warning(f"Tried to set invalid parent on {self}: (got: {parent})")
+            log.warning(f"Tried to set invalid parent on {self}: (got: {repr(parent)} ({type(parent)}))")
 
     @property
     def parent_id(self):
@@ -1045,6 +1045,9 @@ def sanitize_data(self, data):
         blob = None
         try:
             self._data_path = Path(data["path"])
+            # prepend the scan's home dir if the path is relative
+            if not self._data_path.is_absolute():
+                self._data_path = self.scan.home / self._data_path
             if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:

bbot/modules/filedownload.py

@@ -179,7 +179,9 @@ def make_filename(self, url, content_type=None):
         if extension:
             filename = f"{filename}.{extension}"
             orig_filename = f"{orig_filename}.{extension}"
-        return orig_filename, self.download_dir / filename, base_url
+        file_destination = self.download_dir / filename
+        file_destination = self.helpers.truncate_filename(file_destination)
+        return orig_filename, file_destination, base_url
 
     async def report(self):
         if self.files_downloaded > 0:

bbot/modules/gowitness.py

@@ -142,6 +142,9 @@ async def handle_batch(self, *events):
             url = screenshot["url"]
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
+            filename = filename.relative_to(self.scan.home)
+            # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
+            filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
             parent_event = event_dict[url]
             await self.emit_event(

bbot/test/test_step_2/module_tests/test_module_filedownload.py

@@ -60,3 +60,28 @@ def check(self, module_test, events):
         # we don't want html files
         html_files = list(download_dir.glob("*.html"))
         assert len(html_files) == 0, "HTML files were erroneously downloaded"
+
+
+class TestFileDownloadLongFilename(TestFileDownload):
+    async def setup_after_prep(self, module_test):
+        module_test.set_expect_requests(
+            {"uri": "/"},
+            {
+                "response_data": '<a href="/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity.txt"/>'
+            },
+        )
+        module_test.set_expect_requests(
+            {
+                "uri": "/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity.txt"
+            },
+            {
+                "response_data": "juicy stuff",
+            },
+        )
+
+    def check(self, module_test, events):
+        filesystem_events = [e for e in events if e.type == "FILESYSTEM"]
+        assert len(filesystem_events) == 1
+        file_path = Path(filesystem_events[0].data["path"])
+        assert file_path.is_file(), f"File not found at {file_path}"
+        assert file_path.read_text() == "juicy stuff", f"File at {file_path} does not contain the correct content"

bbot/test/test_step_2/module_tests/test_module_gowitness.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from .base import ModuleTestBase
 
 
@@ -102,3 +104,33 @@ def check(self, module_test, events):
         webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"]
         assert webscreenshots, "failed to raise WEBSCREENSHOT events"
         assert all("blob" in e.data and e.data["blob"] for e in webscreenshots), "blob not found in WEBSCREENSHOT data"
+
+
+class TestGoWitnessLongFilename(TestGowitness):
+    """
+    Make sure long filenames are truncated properly
+    """
+
+    targets = [
+        "http://127.0.0.1:8888/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity"
+    ]
+    config_overrides = {"file_blobs": True}
+
+    async def setup_after_prep(self, module_test):
+        request_args = {
+            "uri": "/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity"
+        }
+        respond_args = {
+            "response_data": "<html><head><title>BBOT is life</title></head><body>BBOT is life</body></html>",
+            "headers": {"Server": "Apache/2.4.41 (Ubuntu)"},
+        }
+        module_test.set_expect_requests(request_args, respond_args)
+
+    def check(self, module_test, events):
+        webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"]
+        assert webscreenshots, "failed to raise WEBSCREENSHOT events"
+        assert len(webscreenshots) == 1
+        webscreenshot = webscreenshots[0]
+        filename = Path(webscreenshot.data["path"])
+        # sadly this file doesn't exist because gowitness doesn't truncate properly
+        assert not filename.exists()