From bfe4d01a6d938e24d9897ebcc8665c320b106046 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 31 Oct 2023 11:26:07 -0400 Subject: [PATCH] Fix inconsistency with 404 URLs --- bbot/modules/deadly/nuclei.py | 4 +++- bbot/modules/httpx.py | 3 ++- .../module_tests/test_module_httpx.py | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 94d664b6f..df5f86140 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -175,7 +175,9 @@ def correlate_event(self, events, host): for event in events: if host in event: return event - self.warning("Failed to correlate nuclei result with event") + self.verbose(f"Failed to correlate nuclei result for {host}. Possible source events:") + for event in events: + self.verbose(f" - {event.data}") async def execute_nuclei(self, nuclei_input): command = [ diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index ff53e9972..6341f16ea 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -129,7 +129,8 @@ async def handle_batch(self, *events): continue # discard 404s from unverified URLs - if source_event.type == "URL_UNVERIFIED" and status_code in (404,): + path = j.get("path", "/") + if source_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/": self.debug(f'Discarding 404 from "{url}"') continue diff --git a/bbot/test/test_step_2/module_tests/test_module_httpx.py b/bbot/test/test_step_2/module_tests/test_module_httpx.py index a58fb3133..fcf134dd3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_2/module_tests/test_module_httpx.py @@ -48,3 +48,21 @@ def check(self, module_test, events): url = True assert url, "Failed to visit target URL" assert open_port, "Failed to visit target OPEN_TCP_PORT" + + +class TestHTTPX_404(ModuleTestBase): + targets = ["https://127.0.0.1:9999"] + modules_overrides = ["httpx", "speculate", "excavate"] + config_overrides = {"internal_modules": {"speculate": {"ports": "8888,9999"}}} + + async def setup_after_prep(self, module_test): + module_test.httpserver.expect_request("/").respond_with_data( + "Redirecting...", status=301, headers={"Location": "https://127.0.0.1:9999"} + ) + module_test.httpserver_ssl.expect_request("/").respond_with_data("404 not found", status=404) + + def check(self, module_test, events): + assert 1 == len( + [e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and "status-301" in e.tags] + ) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "https://127.0.0.1:9999/"])