From 40b034b5b6e3c8f46ac3737c71594fb7f166a5b7 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 23 Aug 2024 20:06:38 -0400 Subject: [PATCH 1/2] fixing web_parameters appearing when no modules listening --- bbot/modules/internal/excavate.py | 58 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index bf8b22516..e542aa20a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -857,33 +857,35 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon decoded_data = await self.helpers.re.recursive_decode(data) - content_type_lower = content_type.lower() if content_type else "" - extraction_map = { - "json": self.helpers.extract_params_json, - "xml": self.helpers.extract_params_xml, - } - - for source_type, extract_func in extraction_map.items(): - if source_type in content_type_lower: - results = extract_func(data) - if results: - for parameter_name, original_value in results: - description = ( - f"HTTP Extracted Parameter (speculative from {source_type} content) [{parameter_name}]" - ) - data = { - "host": str(event.host), - "type": "SPECULATIVE", - "name": parameter_name, - "original_value": original_value, - "url": str(event.data["url"]), - "additional_params": {}, - "assigned_cookies": self.assigned_cookies, - "description": description, - } - context = f"excavate's Parameter extractor found a speculative WEB_PARAMETER: {parameter_name} by parsing {source_type} data from {str(event.host)}" - await self.emit_event(data, "WEB_PARAMETER", event, context=context) - return + if self.parameter_extraction: + + content_type_lower = content_type.lower() if content_type else "" + extraction_map = { + "json": self.helpers.extract_params_json, + "xml": self.helpers.extract_params_xml, + } + + for source_type, extract_func in extraction_map.items(): + if source_type in content_type_lower: + results = extract_func(data) + if results: + for parameter_name, original_value in results: + description = ( + f"HTTP Extracted Parameter (speculative from {source_type} content) [{parameter_name}]" + ) + data = { + "host": str(event.host), + "type": "SPECULATIVE", + "name": parameter_name, + "original_value": original_value, + "url": str(event.data["url"]), + "additional_params": {}, + "assigned_cookies": self.assigned_cookies, + "description": description, + } + context = f"excavate's Parameter extractor found a speculative WEB_PARAMETER: {parameter_name} by parsing {source_type} data from {str(event.host)}" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + return for result in self.yara_rules.match(data=f"{data}\n{decoded_data}"): rule_name = result.rule @@ -938,7 +940,7 @@ async def handle_event(self, event): for header, header_values in headers.items(): for header_value in header_values: - if header.lower() == "set-cookie": + if header.lower() == "set-cookie" and self.parameter_extraction: if "=" not in header_value: self.debug(f"Cookie found without '=': {header_value}") continue From 2e88ac4a3bf1e48358f8439cef869ec04ab1baa3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 23 Aug 2024 20:48:55 -0400 Subject: [PATCH 2/2] fixing excavate header parameter test --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 576e1de33..3279d3c5d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -856,7 +856,7 @@ def check(self, module_test, events): class TestExcavateHeaders(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["excavate", "httpx"] + modules_overrides = ["excavate", "httpx", "hunt"] config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} async def setup_before_prep(self, module_test):