test blacklist

bbot/scanner/manager.py

@@ -216,9 +216,6 @@ async def _emit_event(self, event, **kwargs):
             event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event)
             if event_blacklisted:
                 event.add_tag("blacklisted")
-
-            # Blacklist purging
-            if "blacklisted" in event.tags:
                 reason = "event host"
                 if event_blacklisted_dns:
                     reason = "DNS associations"

bbot/scanner/scanner.py

@@ -771,12 +771,6 @@ def make_event(self, *args, **kwargs):
         event = make_event(*args, **kwargs)
         return event
 
-    @property
-    def log(self):
-        if self._log is None:
-            self._log = logging.getLogger(f"bbot.agent.scanner")
-        return self._log
-
     @property
     def root_event(self):
         """
@@ -1029,19 +1023,6 @@ async def _status_ticker(self, interval=15):
                 await asyncio.sleep(interval)
                 self.manager.modules_status(_log=True)
 
-    @contextlib.contextmanager
-    def _catch(self, context="scan", finally_callback=None):
-        """
-        Handle common errors by stopping scan, logging tracebacks, etc.
-
-        with catch():
-            do_stuff()
-        """
-        try:
-            yield
-        except BaseException as e:
-            self._handle_exception(e, context=context)
-
     @contextlib.asynccontextmanager
     async def _acatch(self, context="scan", finally_callback=None):
         """

bbot/test/test_step_1/test_manager_scope_accuracy.py

@@ -762,3 +762,31 @@ def custom_setup(scan):
         assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
         assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
         assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
+
+
+@pytest.mark.asyncio
+async def test_manager_blacklist(bbot_config, bbot_scanner, bbot_httpserver, caplog):
+
+    bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="<a href='http://www-prod.test.notreal:8888'/><a href='http://www-dev.test.notreal:8888'/>")
+
+    # dns search distance = 1, report distance = 0
+    config = {"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0}
+    merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(config))
+    scan = bbot_scanner(
+        "http://127.0.0.1:8888",
+        modules=["httpx", "excavate"],
+        config=merged_config,
+        whitelist=["127.0.0.0/29", "test.notreal"],
+        blacklist=["127.0.0.64/29"],
+    )
+    scan.helpers.dns.mock_dns({
+        ("www-prod.test.notreal", "A"): "127.0.0.66",
+        ("www-dev.test.notreal", "A"): "127.0.0.22",
+    })
+
+    events = [e async for e in scan.async_start()]
+
+    assert any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-dev.test.notreal:8888/"])
+    assert not any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-prod.test.notreal:8888/"])
+
+    assert 'Omitting due to blacklisted DNS associations: URL_UNVERIFIED("http://www-prod.test.notreal:8888/"' in caplog.text