From 0777493039525c9e1dcab96d7ef0f6894724861e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 16 Nov 2023 12:32:28 -0500 Subject: [PATCH 1/5] softer warnings for soft-failed modules --- bbot/scanner/manager.py | 28 ++++++++++++++++++++++++ bbot/scanner/scanner.py | 33 +++++++++++++---------------- bbot/test/test_step_1/test_agent.py | 9 +++++++- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 13d46669d..02c78a7e9 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -175,6 +175,8 @@ async def _emit_event(self, event, **kwargs): on_success_callback = kwargs.pop("on_success_callback", None) abort_if = kwargs.pop("abort_if", None) + log.debug(f"EMIT {event} 1") + # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist if skip_dns_resolution: @@ -198,6 +200,8 @@ async def _emit_event(self, event, **kwargs): for ip in ips: resolved_hosts.add(ip) + log.debug(f"EMIT {event} 2") + # kill runaway DNS chains dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) if dns_resolve_distance >= self.scan.helpers.dns.max_dns_resolve_distance: @@ -206,6 +210,8 @@ async def _emit_event(self, event, **kwargs): ) dns_children = {} + log.debug(f"EMIT {event} 3") + if event.type in ("DNS_NAME", "IP_ADDRESS"): for tag in dns_tags: event.add_tag(tag) @@ -222,6 +228,8 @@ async def _emit_event(self, event, **kwargs): log.debug(f"Omitting due to blacklisted {reason}: {event}") return + log.debug(f"EMIT {event} 4") + # DNS_NAME --> DNS_NAME_UNRESOLVED if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: event.type = "DNS_NAME_UNRESOLVED" @@ -229,6 +237,8 @@ async def _emit_event(self, event, **kwargs): # Cloud tagging await self.scan.helpers.cloud.tag_event(event) + log.debug(f"EMIT {event} 5") + # Scope shepherding # here is where we make sure in-scope events are set to their proper scope distance if event.host and event_whitelisted: @@ -243,18 +253,24 @@ async def _emit_event(self, event, **kwargs): ) event.internal = True + log.debug(f"EMIT {event} 6") + # check for wildcards if event.scope_distance <= self.scan.scope_search_distance: if not "unresolved" in event.tags: if not self.scan.helpers.is_ip_type(event.host): await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) + log.debug(f"EMIT {event} 7") + # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() if event.type == "DNS_NAME": acceptable = self._event_precheck(event) if not acceptable: return + log.debug(f"EMIT {event} 8") + # if we discovered something interesting from an internal event, # make sure we preserve its chain of parents source = event.source @@ -267,6 +283,8 @@ async def _emit_event(self, event, **kwargs): log.debug(f"Re-queuing internal event {source} with parent {event}") self.queue_event(source) + log.debug(f"EMIT {event} 9") + # now that the event is properly tagged, we can finally make decisions about it abort_result = False if callable(abort_if): @@ -280,14 +298,20 @@ async def _emit_event(self, event, **kwargs): log.debug(msg) return + log.debug(f"EMIT {event} 10") + # run success callback before distributing event (so it can add tags, etc.) if callable(on_success_callback): async with self.scan._acatch(context=on_success_callback): await self.scan.helpers.execute_sync_or_async(on_success_callback, event) + log.debug(f"EMIT {event} 11") + await self.distribute_event(event) event_distributed = True + log.debug(f"EMIT {event} 12") + # speculate DNS_NAMES and IP_ADDRESSes from other event types source_event = event if ( @@ -305,6 +329,8 @@ async def _emit_event(self, event, **kwargs): source_event.add_tag("target") self.queue_event(source_event) + log.debug(f"EMIT {event} 13") + ### Emit DNS children ### if self.dns_resolution: emit_children = True @@ -336,6 +362,8 @@ async def _emit_event(self, event, **kwargs): for child_event in dns_child_events: self.queue_event(child_event) + log.debug(f"EMIT {event} 14") + except ValidationError as e: log.warning(f"Event validation failed with kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 84c623300..fbf18b609 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -286,9 +286,20 @@ async def _prep(self): await self.load_modules() self.info(f"Setting up modules...") - await self.setup_modules() + succeeded, hard_failed, soft_failed = await self.setup_modules() + + num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) + if num_output_modules < 1: + raise ScanError("Failed to load output modules. Aborting.") + total_failed = len(hard_failed + soft_failed) + if hard_failed: + msg = f"Setup hard-failed for {len(hard_failed):,} modules ({','.join(hard_failed)})" + self._fail_setup(msg) + + total_modules = total_failed + len(self.modules) + success_msg = f"Setup succeeded for {len(self.modules):,}/{total_modules:,} modules." - self.success(f"Setup succeeded for {len(self.modules):,} modules.") + self.success(success_msg) self._prepped = True def start(self): @@ -443,26 +454,12 @@ async def setup_modules(self, remove_failed=True): self.modules[module_name].set_error_state() hard_failed.append(module_name) else: - self.warning(f"Setup soft-failed for {module_name}: {msg}") + self.info(f"Setup soft-failed for {module_name}: {msg}") soft_failed.append(module_name) if not status and remove_failed: self.modules.pop(module_name) - num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) - if num_output_modules < 1: - raise ScanError("Failed to load output modules. Aborting.") - total_failed = len(hard_failed + soft_failed) - if hard_failed: - msg = f"Setup hard-failed for {len(hard_failed):,} modules ({','.join(hard_failed)})" - self._fail_setup(msg) - elif total_failed > 0: - self.warning(f"Setup failed for {total_failed:,} modules") - - return { - "succeeded": succeeded, - "hard_failed": hard_failed, - "soft_failed": soft_failed, - } + return succeeded, hard_failed, soft_failed async def load_modules(self): """Asynchronously import and instantiate all scan modules, including internal and output modules. diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py index 73bb50355..444cfdb16 100644 --- a/bbot/test/test_step_1/test_agent.py +++ b/bbot/test/test_step_1/test_agent.py @@ -140,7 +140,7 @@ async def test_agent(agent): global success async with websockets.serve(_websocket_handler, "127.0.0.1", 8765): - asyncio.create_task(agent.start()) + agent_task = asyncio.create_task(agent.start()) # wait for 30 seconds await asyncio.wait_for(scan_done.wait(), 30) assert success @@ -149,3 +149,10 @@ async def test_agent(agent): await agent.start_scan("scan_to_be_rejected", targets=["127.0.0.1"], modules=["ipneighbor"]) await asyncio.sleep(0.1) await agent.stop_scan() + tasks = [agent.task, agent_task] + for task in tasks: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass From 9c9ec7209502cf6300e1855bc914676b8122be19 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 16 Nov 2023 13:37:08 -0500 Subject: [PATCH 2/5] raise agent test timeout --- bbot/scanner/scanner.py | 10 +++++----- bbot/test/test_step_1/test_agent.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index fbf18b609..859d3de82 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -426,10 +426,10 @@ async def setup_modules(self, remove_failed=True): remove_failed (bool): Flag indicating whether to remove modules that fail setup. Returns: - dict: Dictionary containing lists of module names categorized by their setup status. - 'succeeded' - List of modules that successfully set up. - 'hard_failed' - List of modules that encountered a hard failure during setup. - 'soft_failed' - List of modules that encountered a soft failure during setup. + tuple: + succeeded - List of modules that successfully set up. + hard_failed - List of modules that encountered a hard failure during setup. + soft_failed - List of modules that encountered a soft failure during setup. Raises: ScanError: If no output modules could be loaded. @@ -450,7 +450,7 @@ async def setup_modules(self, remove_failed=True): self.debug(f"Setup succeeded for {module_name} ({msg})") succeeded.append(module_name) elif status == False: - self.error(f"Setup hard-failed for {module_name}: {msg}") + self.warning(f"Setup hard-failed for {module_name}: {msg}") self.modules[module_name].set_error_state() hard_failed.append(module_name) else: diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py index 444cfdb16..32a1b33fb 100644 --- a/bbot/test/test_step_1/test_agent.py +++ b/bbot/test/test_step_1/test_agent.py @@ -141,8 +141,8 @@ async def test_agent(agent): global success async with websockets.serve(_websocket_handler, "127.0.0.1", 8765): agent_task = asyncio.create_task(agent.start()) - # wait for 30 seconds - await asyncio.wait_for(scan_done.wait(), 30) + # wait for 90 seconds + await asyncio.wait_for(scan_done.wait(), 90) assert success await agent.start_scan("scan_to_be_cancelled", targets=["127.0.0.1"], modules=["ipneighbor"]) From 2fb0162e29c909d479efc57788bb7f30bd994d90 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 16 Nov 2023 13:58:20 -0500 Subject: [PATCH 3/5] test debugging --- bbot/test/test_step_1/test_agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py index 32a1b33fb..b492f312c 100644 --- a/bbot/test/test_step_1/test_agent.py +++ b/bbot/test/test_step_1/test_agent.py @@ -31,7 +31,7 @@ async def websocket_handler(websocket, path, scan_done=None): assert websocket.request_headers["Authorization"] == "Bearer test" async for message in websocket: - log.debug(f"PHASE: {phase}, MESSAGE: {message}") + log.critical(f"PHASE: {phase}, MESSAGE: {message}") if not control or not first_run: continue m = json.loads(message) @@ -142,7 +142,7 @@ async def test_agent(agent): async with websockets.serve(_websocket_handler, "127.0.0.1", 8765): agent_task = asyncio.create_task(agent.start()) # wait for 90 seconds - await asyncio.wait_for(scan_done.wait(), 90) + await asyncio.wait_for(scan_done.wait(), 60) assert success await agent.start_scan("scan_to_be_cancelled", targets=["127.0.0.1"], modules=["ipneighbor"]) @@ -151,8 +151,8 @@ async def test_agent(agent): await agent.stop_scan() tasks = [agent.task, agent_task] for task in tasks: - task.cancel() try: + task.cancel() await task - except asyncio.CancelledError: + except (asyncio.CancelledError, AttributeError): pass From 0297e2c578aeaa1c8e3d1b1b504417190662aa62 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 16 Nov 2023 14:44:20 -0500 Subject: [PATCH 4/5] use fake domain in excavate tests --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 18d453b03..2d65dde4d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -212,7 +212,7 @@ def check(self, module_test, events): class TestExcavateCSP(TestExcavate): - csp_test_header = "default-src 'self'; script-src fake.domain.com; object-src 'none';" + csp_test_header = "default-src 'self'; script-src test.asdf.fakedomain; object-src 'none';" async def setup_before_prep(self, module_test): expect_args = {"method": "GET", "uri": "/"} @@ -220,4 +220,4 @@ async def setup_before_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert any(e.data == "fake.domain.com" for e in events) + assert any(e.data == "test.asdf.fakedomain" for e in events) From cb7c9a98c7f5904615a0b6658876c72031574502 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 16 Nov 2023 14:47:48 -0500 Subject: [PATCH 5/5] remove debug statements --- bbot/scanner/manager.py | 28 ---------------------------- bbot/test/test_step_1/test_agent.py | 2 +- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 02c78a7e9..13d46669d 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -175,8 +175,6 @@ async def _emit_event(self, event, **kwargs): on_success_callback = kwargs.pop("on_success_callback", None) abort_if = kwargs.pop("abort_if", None) - log.debug(f"EMIT {event} 1") - # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist if skip_dns_resolution: @@ -200,8 +198,6 @@ async def _emit_event(self, event, **kwargs): for ip in ips: resolved_hosts.add(ip) - log.debug(f"EMIT {event} 2") - # kill runaway DNS chains dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) if dns_resolve_distance >= self.scan.helpers.dns.max_dns_resolve_distance: @@ -210,8 +206,6 @@ async def _emit_event(self, event, **kwargs): ) dns_children = {} - log.debug(f"EMIT {event} 3") - if event.type in ("DNS_NAME", "IP_ADDRESS"): for tag in dns_tags: event.add_tag(tag) @@ -228,8 +222,6 @@ async def _emit_event(self, event, **kwargs): log.debug(f"Omitting due to blacklisted {reason}: {event}") return - log.debug(f"EMIT {event} 4") - # DNS_NAME --> DNS_NAME_UNRESOLVED if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: event.type = "DNS_NAME_UNRESOLVED" @@ -237,8 +229,6 @@ async def _emit_event(self, event, **kwargs): # Cloud tagging await self.scan.helpers.cloud.tag_event(event) - log.debug(f"EMIT {event} 5") - # Scope shepherding # here is where we make sure in-scope events are set to their proper scope distance if event.host and event_whitelisted: @@ -253,24 +243,18 @@ async def _emit_event(self, event, **kwargs): ) event.internal = True - log.debug(f"EMIT {event} 6") - # check for wildcards if event.scope_distance <= self.scan.scope_search_distance: if not "unresolved" in event.tags: if not self.scan.helpers.is_ip_type(event.host): await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) - log.debug(f"EMIT {event} 7") - # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() if event.type == "DNS_NAME": acceptable = self._event_precheck(event) if not acceptable: return - log.debug(f"EMIT {event} 8") - # if we discovered something interesting from an internal event, # make sure we preserve its chain of parents source = event.source @@ -283,8 +267,6 @@ async def _emit_event(self, event, **kwargs): log.debug(f"Re-queuing internal event {source} with parent {event}") self.queue_event(source) - log.debug(f"EMIT {event} 9") - # now that the event is properly tagged, we can finally make decisions about it abort_result = False if callable(abort_if): @@ -298,20 +280,14 @@ async def _emit_event(self, event, **kwargs): log.debug(msg) return - log.debug(f"EMIT {event} 10") - # run success callback before distributing event (so it can add tags, etc.) if callable(on_success_callback): async with self.scan._acatch(context=on_success_callback): await self.scan.helpers.execute_sync_or_async(on_success_callback, event) - log.debug(f"EMIT {event} 11") - await self.distribute_event(event) event_distributed = True - log.debug(f"EMIT {event} 12") - # speculate DNS_NAMES and IP_ADDRESSes from other event types source_event = event if ( @@ -329,8 +305,6 @@ async def _emit_event(self, event, **kwargs): source_event.add_tag("target") self.queue_event(source_event) - log.debug(f"EMIT {event} 13") - ### Emit DNS children ### if self.dns_resolution: emit_children = True @@ -362,8 +336,6 @@ async def _emit_event(self, event, **kwargs): for child_event in dns_child_events: self.queue_event(child_event) - log.debug(f"EMIT {event} 14") - except ValidationError as e: log.warning(f"Event validation failed with kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py index b492f312c..00d70a751 100644 --- a/bbot/test/test_step_1/test_agent.py +++ b/bbot/test/test_step_1/test_agent.py @@ -31,7 +31,7 @@ async def websocket_handler(websocket, path, scan_done=None): assert websocket.request_headers["Authorization"] == "Bearer test" async for message in websocket: - log.critical(f"PHASE: {phase}, MESSAGE: {message}") + log.debug(f"PHASE: {phase}, MESSAGE: {message}") if not control or not first_run: continue m = json.loads(message)