Merge pull request #2065 from blacklanternsecurity/cloudcheck-temp-fix

Performance boost - cloudcheck, httpx

bbot/modules/httpx.py

@@ -1,5 +1,5 @@
 import re
-import json
+import orjson
 import tempfile
 import subprocess
 from pathlib import Path
@@ -142,11 +142,11 @@ async def handle_batch(self, *events):
         proxy = self.scan.http_proxy
         if proxy:
             command += ["-http-proxy", proxy]
-        async for line in self.run_process_live(command, input=list(stdin), stderr=subprocess.DEVNULL):
+        async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL):
             try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
+                j = await self.helpers.run_in_executor(orjson.loads, line)
+            except orjson.JSONDecodeError:
+                self.warning(f"httpx failed to decode line: {line}")
                 continue
 
             url = j.get("url", "")

bbot/modules/internal/cloudcheck.py

@@ -72,9 +72,10 @@ async def handle_event(self, event, **kwargs):
                 base_kwargs["event_type"] = event_type
                 for sig in sigs:
                     matches = []
-                    if event.type == "HTTP_RESPONSE":
-                        matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
-                    elif event.type.startswith("DNS_NAME"):
+                    # TODO: convert this to an excavate YARA hook
+                    # if event.type == "HTTP_RESPONSE":
+                    #     matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
+                    if event.type.startswith("DNS_NAME"):
                         for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:

bbot/test/test_step_2/module_tests/test_module_cloudcheck.py

@@ -8,7 +8,7 @@ class TestCloudCheck(ModuleTestBase):
     modules_overrides = ["httpx", "excavate", "cloudcheck"]
 
     async def setup_after_prep(self, module_test):
-        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='asdf.s3.amazonaws.com'/>"})
+        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='http://asdf.s3.amazonaws.com'/>"})
 
         scan = Scanner(config={"cloudcheck": True})
         await scan._prep()

pyproject.toml

@@ -56,6 +56,7 @@ httpx = "^0.27.0"
 puremagic = "^1.28"
 cloudcheck = "^6.0.0.602"
 radixtarget = "^2.0.0.50"
+orjson = "^3.10.12"
 
 [tool.poetry.group.dev.dependencies]
 poetry-dynamic-versioning = ">=0.21.4,<1.5.0"