Merge pull request #808 from blacklanternsecurity/fix-404-inconsistency

Fix inconsistency with 404 URLs

bbot/modules/deadly/nuclei.py

@@ -175,7 +175,9 @@ def correlate_event(self, events, host):
         for event in events:
             if host in event:
                 return event
-        self.warning("Failed to correlate nuclei result with event")
+        self.verbose(f"Failed to correlate nuclei result for {host}. Possible source events:")
+        for event in events:
+            self.verbose(f" - {event.data}")
 
     async def execute_nuclei(self, nuclei_input):
         command = [

bbot/modules/httpx.py

@@ -129,7 +129,8 @@ async def handle_batch(self, *events):
                 continue
 
             # discard 404s from unverified URLs
-            if source_event.type == "URL_UNVERIFIED" and status_code in (404,):
+            path = j.get("path", "/")
+            if source_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/":
                 self.debug(f'Discarding 404 from "{url}"')
                 continue
 

bbot/test/test_step_2/module_tests/test_module_httpx.py

@@ -48,3 +48,21 @@ def check(self, module_test, events):
                     url = True
         assert url, "Failed to visit target URL"
         assert open_port, "Failed to visit target OPEN_TCP_PORT"
+
+
+class TestHTTPX_404(ModuleTestBase):
+    targets = ["https://127.0.0.1:9999"]
+    modules_overrides = ["httpx", "speculate", "excavate"]
+    config_overrides = {"internal_modules": {"speculate": {"ports": "8888,9999"}}}
+
+    async def setup_after_prep(self, module_test):
+        module_test.httpserver.expect_request("/").respond_with_data(
+            "Redirecting...", status=301, headers={"Location": "https://127.0.0.1:9999"}
+        )
+        module_test.httpserver_ssl.expect_request("/").respond_with_data("404 not found", status=404)
+
+    def check(self, module_test, events):
+        assert 1 == len(
+            [e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and "status-301" in e.tags]
+        )
+        assert 1 == len([e for e in events if e.type == "URL" and e.data == "https://127.0.0.1:9999/"])