small tweaks, updated tests to use local server

bbot/core/helpers/dns.py

@@ -1004,6 +1004,8 @@ async def _connectivity_check(self, interval=5):
                 return True
         dns_server_working = []
         async with self._dns_connectivity_lock:
+            if time.time() - self._last_dns_success < interval:
+                return True
             with suppress(Exception):
                 dns_server_working = await self._catch(self.resolver.resolve, "www.google.com", rdtype="A")
                 if dns_server_working:

bbot/modules/deadly/dastardly.py

@@ -3,73 +3,94 @@
 
 
 class dastardly(BaseModule):
-    watched_events = ["URL"]
+    watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING", "VULNERABILITY"]
-    flags = ["active", "aggressive"]
+    flags = ["active", "aggressive", "slow", "web-thorough"]
     meta = {"description": "Lightweight web application security scanner"}
 
-    deps_apt = ["docker.io"]
     deps_pip = ["lxml~=4.9.2"]
-    deps_shell = ["docker pull public.ecr.aws/portswigger/dastardly:latest"]
-    in_scope_only = True
+    deps_ansible = [
+        {
+            "name": "Install Docker (Non-Debian)",
+            "package": {"name": "docker", "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] != 'Debian'",
+        },
+        {
+            "name": "Install Docker (Debian)",
+            "package": {
+                "name": "docker.io",
+                "state": "present",
+            },
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Debian'",
+        },
+    ]
+    per_host_only = True
 
     async def setup(self):
-        self.helpers.depsinstaller.ensure_root(message="Dastardly: docker requires root privileges")
+        await self.helpers.run("systemctl", "start", "docker", sudo=True)
+        await self.helpers.run("docker", "pull", "public.ecr.aws/portswigger/dastardly:latest", sudo=True)
+        self.output_dir = self.scan.home / "dastardly"
+        self.helpers.mkdir(self.output_dir)
+        return True
+
+    async def filter_event(self, event):
+        # Reject redirects. This helps to avoid scanning the same site twice.
+        is_redirect = str(event.data["status_code"]).startswith("30")
+        if is_redirect:
+            return False, "URL is a redirect"
         return True
 
     async def handle_event(self, event):
-        host = str(event.data)
+        host = event.parsed._replace(path="/").geturl()
+        self.verbose(f"Running Dastardly scan against {host}")
         command, output_file = self.construct_command(host)
-        try:
-            await self.helpers.run(command, sudo=True)
-            for testsuite in self.parse_dastardly_xml(output_file):
-                url = testsuite.endpoint
-                for testcase in testsuite.testcases:
-                    for failure in testcase.failures:
-                        message = failure.instance
-                        detail = failure.text
-                        if failure.severity == "Info":
-                            self.emit_event(
-                                {
-                                    "host": str(event.host),
-                                    "url": url,
-                                    "description": message,
-                                    "detail": detail,
-                                },
-                                "FINDING",
-                                event,
-                            )
-                        else:
-                            self.emit_event(
-                                {
-                                    "severity": failure.severity,
-                                    "host": str(event.host),
-                                    "url": url,
-                                    "description": message,
-                                    "detail": detail,
-                                },
-                                "VULNERABILITY",
-                                event,
-                            )
-        finally:
-            output_file.unlink(missing_ok=True)
+        finished_proc = await self.helpers.run(command, sudo=True)
+        self.debug(f'dastardly stdout: {getattr(finished_proc, "stdout", "")}')
+        self.debug(f'dastardly stderr: {getattr(finished_proc, "stderr", "")}')
+        for testsuite in self.parse_dastardly_xml(output_file):
+            url = testsuite.endpoint
+            for testcase in testsuite.testcases:
+                for failure in testcase.failures:
+                    if failure.severity == "Info":
+                        self.emit_event(
+                            {
+                                "host": str(event.host),
+                                "url": url,
+                                "description": failure.instance,
+                            },
+                            "FINDING",
+                            event,
+                        )
+                    else:
+                        self.emit_event(
+                            {
+                                "severity": failure.severity,
+                                "host": str(event.host),
+                                "url": url,
+                                "description": failure.instance,
+                            },
+                            "VULNERABILITY",
+                            event,
+                        )
 
     def construct_command(self, target):
-        temp_path = self.helpers.temp_filename(extension="xml")
-        filename = temp_path.name
-        temp_dir = temp_path.parent
+        date_time = self.helpers.make_date()
+        file_name = self.helpers.tagify(target)
+        temp_path = self.output_dir / f"{date_time}_{file_name}.xml"
         command = [
             "docker",
             "run",
             "--user",
             "0",
             "--rm",
             "-v",
-            f"{temp_dir}:/dastardly",
+            f"{self.output_dir}:/dastardly",
             "-e",
             f"BURP_START_URL={target}",
             "-e",
-            f"BURP_REPORT_FILE_PATH=/dastardly/{filename}",
+            f"BURP_REPORT_FILE_PATH=/dastardly/{temp_path.name}",
             "public.ecr.aws/portswigger/dastardly:latest",
         ]
         return command, temp_path
@@ -83,10 +104,6 @@ def parse_dastardly_xml(self, xml_file):
         except Exception as e:
             self.warning(f"Error parsing Dastardly XML at {xml_file}: {e}")
 
-    async def cleanup(self):
-        resume_file = self.helpers.current_dir / "resume.cfg"
-        resume_file.unlink(missing_ok=True)
-
 
 class Failure:
     def __init__(self, xml):

bbot/modules/secretsdb.py

@@ -42,6 +42,7 @@ async def setup(self):
         return True
 
     async def handle_event(self, event):
+        self.critical(event)
         resp_body = event.data.get("body", "")
         resp_headers = event.data.get("raw_header", "")
         all_matches = await self.scan.run_in_executor(self.search_data, resp_body, resp_headers)

bbot/test/conftest.py

@@ -35,7 +35,7 @@ def assert_all_responses_were_requested() -> bool:
 
 @pytest.fixture
 def bbot_httpserver():
-    server = HTTPServer(host="127.0.0.1", port=8888)
+    server = HTTPServer(host="0.0.0.0", port=8888)
     server.start()
 
     yield server

bbot/test/test_step_2/module_tests/test_module_dastardly.py

@@ -1,19 +1,63 @@
+import json
+from werkzeug import Response
+
 from .base import ModuleTestBase
 
 
 class TestDastardly(ModuleTestBase):
-    targets = ["ginandjuice.shop"]
-    modules_overrides = ["nmap", "httpx", "dastardly"]
+    targets = ["http://127.0.0.1:8888/"]
+    modules_overrides = ["httpx", "dastardly"]
+
+    web_response = """<!DOCTYPE html>
+    <html>
+    <body>
+        <a href="/test?test=yes">visit this<a/>
+    </body>
+    </html>"""
+
+    def xss_handler(self, request):
+        response = f"""<!DOCTYPE html>
+    <html>
+    <head>
+        <title>Email Form</title>
+    </head>
+    <body>
+        {request.args.get("test", "")}
+    </body>
+    </html>"""
+        return Response(response, content_type="text/html")
+
+    async def get_docker_ip(self, module_test):
+        docker_ip = "172.17.0.1"
+        try:
+            ip_output = await module_test.scan.helpers.run(["ip", "-j", "-4", "a", "show", "dev", "docker0"])
+            interface_json = json.loads(ip_output.stdout)
+            docker_ip = interface_json[0]["addr_info"][0]["local"]
+        except Exception:
+            pass
+        return docker_ip
+
+    async def setup_after_prep(self, module_test):
+        module_test.httpserver.expect_request("/").respond_with_data(self.web_response)
+        module_test.httpserver.expect_request("/test").respond_with_handler(self.xss_handler)
+
+        # get docker IP
+        docker_ip = await self.get_docker_ip(module_test)
+        module_test.scan.target.add_target(docker_ip)
+
+        # replace 127.0.0.1 with docker host IP to allow dastardly access to local http server
+        old_filter_event = module_test.module.filter_event
+
+        def new_filter_event(event):
+            self.new_url = f"http://{docker_ip}:8888/"
+            event.data["url"] = self.new_url
+            event.parsed = module_test.scan.helpers.urlparse(self.new_url)
+            return old_filter_event(event)
+
+        module_test.monkeypatch.setattr(module_test.module, "filter_event", new_filter_event)
 
     def check(self, module_test, events):
-        reflected_xss = False
-        vulnerable_js = False
-        for e in events:
-            if e.type == "VULNERABILITY":
-                if "Cross-site scripting (reflected)" in e.data["description"]:
-                    reflected_xss = True
-            if e.type == "VULNERABILITY":
-                if "Vulnerable JavaScript dependency" in e.data["description"]:
-                    vulnerable_js = True
-        assert reflected_xss
-        assert vulnerable_js
+        assert 1 == len([e for e in events if e.type == "VULNERABILITY"])
+        assert 1 == len(
+            [e for e in events if e.type == "VULNERABILITY" and f"{self.new_url}test" in e.data["description"]]
+        )