Merge pull request #1678 from blacklanternsecurity/better-engine-debu…

…gging

Better engine debugging, misc bugfixes

bbot/core/helpers/dns/engine.py

@@ -349,57 +349,20 @@ async def resolve_batch(self, queries, threads=10, **kwargs):
             ('www.evilcorp.com', {'1.1.1.1'})
             ('evilcorp.com', {'2.2.2.2'})
         """
-        tasks = {}
-        client_id = self.client_id_var.get()
-
-        def new_task(query):
-            task = self.new_child_task(client_id, self.resolve(query, **kwargs))
-            tasks[task] = query
-
-        queries = list(queries)
-        for _ in range(threads):  # Start initial batch of tasks
-            if queries:  # Ensure there are args to process
-                new_task(queries.pop(0))
-
-        while tasks:  # While there are tasks pending
-            # Wait for the first task to complete
-            finished = await self.finished_tasks(client_id, timeout=120)
-
-            for task in finished:
-                results = task.result()
-                query = tasks.pop(task)
-
-                if results:
-                    yield (query, results)
-
-                if queries:  # Start a new task for each one completed, if URLs remain
-                    new_task(queries.pop(0))
+        async for (args, _, _), responses in self.task_pool(
+            self.resolve, args_kwargs=queries, threads=threads, global_kwargs=kwargs
+        ):
+            yield args[0], responses
 
     async def resolve_raw_batch(self, queries, threads=10, **kwargs):
-        tasks = {}
-        client_id = self.client_id_var.get()
-
-        def new_task(query, rdtype):
-            task = self.new_child_task(client_id, self.resolve_raw(query, type=rdtype, **kwargs))
-            tasks[task] = (query, rdtype)
-
-        queries = list(queries)
-        for _ in range(threads):  # Start initial batch of tasks
-            if queries:  # Ensure there are args to process
-                new_task(*queries.pop(0))
-
-        while tasks:  # While there are tasks pending
-            # Wait for the first task to complete
-            finished = await self.finished_tasks(client_id, timeout=120)
-
-            for task in finished:
-                answers, errors = task.result()
-                query, rdtype = tasks.pop(task)
-                for answer in answers:
-                    yield ((query, rdtype), (answer, errors))
-
-                if queries:  # Start a new task for each one completed, if URLs remain
-                    new_task(*queries.pop(0))
+        queries_kwargs = [[q[0], {"type": q[1]}] for q in queries]
+        async for (args, kwargs, _), (answers, errors) in self.task_pool(
+            self.resolve_raw, args_kwargs=queries_kwargs, threads=threads, global_kwargs=kwargs
+        ):
+            query = args[0]
+            rdtype = kwargs["type"]
+            for answer in answers:
+                yield ((query, rdtype), (answer, errors))
 
     async def _catch(self, callback, *args, **kwargs):
         """

bbot/core/helpers/web/client.py

@@ -72,6 +72,7 @@ def __init__(self, *args, **kwargs):
         proxies = self._web_config.get("http_proxy", None)
         kwargs["proxies"] = proxies
 
+        log.verbose(f"Creating httpx.AsyncClient({args}, {kwargs})")
         super().__init__(*args, **kwargs)
         if not self._persist_cookies:
             self._cookies = DummyCookies()
@@ -91,3 +92,7 @@ def _merge_cookies(self, cookies):
         if self._persist_cookies:
             return super()._merge_cookies(cookies)
         return cookies
+
+    @property
+    def retries(self):
+        return self._transport._pool._retries

bbot/core/helpers/web/engine.py

@@ -35,20 +35,19 @@ def __init__(self, socket_path, target, config={}, debug=False):
         self.http_debug = self.web_config.get("debug", False)
         self._ssl_context_noverify = None
         self.web_clients = {}
-        self.web_clients[0] = self.AsyncClient(persist_cookies=False, retries=0)
-        self.web_client = self.web_clients[0]
+        self.web_client = self.AsyncClient(persist_cookies=False)
 
     def AsyncClient(self, *args, **kwargs):
         # cache by retries to prevent unwanted accumulation of clients
         # (they are not garbage-collected)
-        retries = kwargs.get("retries", 0)
+        retries = kwargs.get("retries", 1)
         try:
             return self.web_clients[retries]
         except KeyError:
             from .client import BBOTAsyncClient
 
             client = BBOTAsyncClient.from_config(self.config, self.target, *args, **kwargs)
-            self.web_clients[retries] = client
+            self.web_clients[client.retries] = client
             return client
 
     async def request(self, *args, **kwargs):
@@ -84,63 +83,25 @@ async def request(self, *args, **kwargs):
 
         async with self._acatch(url, raise_error):
             if self.http_debug:
-                logstr = f"Web request: {str(args)}, {str(kwargs)}"
-                log.trace(logstr)
+                log.trace(f"Web request: {str(args)}, {str(kwargs)}")
             response = await client.request(*args, **kwargs)
             if self.http_debug:
                 log.trace(
                     f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
                 )
             return response
 
-    async def request_batch(self, urls, *args, threads=10, **kwargs):
-        tasks = {}
-        client_id = self.client_id_var.get()
-
-        urls = list(urls)
-
-        def new_task():
-            if urls:
-                url = urls.pop(0)
-                task = self.new_child_task(client_id, self.request(url, *args, **kwargs))
-                tasks[task] = url
-
-        for _ in range(threads):  # Start initial batch of tasks
-            new_task()
-
-        while tasks:  # While there are tasks pending
-            # Wait for the first task to complete
-            finished = await self.finished_tasks(client_id, timeout=120)
-
-            for task in finished:
-                response = task.result()
-                url = tasks.pop(task)
-                yield (url, response)
-                new_task()
-
-    async def request_custom_batch(self, urls_and_kwargs, threads=10):
-        tasks = {}
-        client_id = self.client_id_var.get()
-        urls_and_kwargs = list(urls_and_kwargs)
-
-        def new_task():
-            if urls_and_kwargs:  # Ensure there are args to process
-                url, kwargs, custom_tracker = urls_and_kwargs.pop(0)
-                task = self.new_child_task(client_id, self.request(url, **kwargs))
-                tasks[task] = (url, kwargs, custom_tracker)
-
-        for _ in range(threads):  # Start initial batch of tasks
-            new_task()
-
-        while tasks:  # While there are tasks pending
-            # Wait for the first task to complete
-            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
-
-            for task in done:
-                response = task.result()
-                url, kwargs, custom_tracker = tasks.pop(task)
-                yield (url, kwargs, custom_tracker, response)
-                new_task()
+    async def request_batch(self, urls, threads=10, **kwargs):
+        async for (args, _, _), response in self.task_pool(
+            self.request, args_kwargs=urls, threads=threads, global_kwargs=kwargs
+        ):
+            yield args[0], response
+
+    async def request_custom_batch(self, urls_and_kwargs, threads=10, **kwargs):
+        async for (args, kwargs, tracker), response in self.task_pool(
+            self.request, args_kwargs=urls_and_kwargs, threads=threads, global_kwargs=kwargs
+        ):
+            yield args[0], kwargs, tracker, response
 
     async def download(self, url, **kwargs):
         warn = kwargs.pop("warn", True)

bbot/core/helpers/web/web.py

@@ -55,6 +55,7 @@ def __init__(self, parent_helper):
         self.web_config = self.config.get("web", {})
         self.web_spider_depth = self.web_config.get("spider_depth", 1)
         self.web_spider_distance = self.web_config.get("spider_distance", 0)
+        self.web_clients = {}
         self.target = self.preset.target
         self.ssl_verify = self.config.get("ssl_verify", False)
         engine_debug = self.config.get("engine", {}).get("debug", False)
@@ -64,9 +65,17 @@ def __init__(self, parent_helper):
         )
 
     def AsyncClient(self, *args, **kwargs):
-        from .client import BBOTAsyncClient
+        # cache by retries to prevent unwanted accumulation of clients
+        # (they are not garbage-collected)
+        retries = kwargs.get("retries", 1)
+        try:
+            return self.web_clients[retries]
+        except KeyError:
+            from .client import BBOTAsyncClient
 
-        return BBOTAsyncClient.from_config(self.config, self.target, *args, persist_cookies=False, **kwargs)
+            client = BBOTAsyncClient.from_config(self.config, self.target, *args, persist_cookies=False, **kwargs)
+            self.web_clients[client.retries] = client
+            return client
 
     async def request(self, *args, **kwargs):
         """

bbot/modules/deadly/ffuf.py

@@ -34,7 +34,7 @@ class ffuf(BaseModule):
     in_scope_only = True
 
     async def setup(self):
-
+        self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist_url = self.config.get("wordlist", "")
         self.debug(f"Using wordlist [{wordlist_url}]")
@@ -243,6 +243,9 @@ async def execute_ffuf(
                 self.debug("invalid mode specified, aborting")
                 return
 
+            if self.proxy:
+                command += ["-x", self.proxy]
+
             if apply_filters:
                 if ext in filters.keys():
                     if filters[ext][0] == ("ABORT"):

bbot/modules/deadly/nuclei.py

@@ -75,7 +75,7 @@ async def setup(self):
                 self.warning(f"Failure while updating nuclei templates: {update_results.stderr}")
         else:
             self.warning("Error running nuclei template update command")
-        self.proxy = self.scan.config.get("http_proxy", "")
+        self.proxy = self.scan.web_config.get("http_proxy", "")
         self.mode = self.config.get("mode", "severe").lower()
         self.ratelimit = int(self.config.get("ratelimit", 150))
         self.concurrency = int(self.config.get("concurrency", 25))

bbot/modules/ffuf_shortnames.py

@@ -67,6 +67,7 @@ class ffuf_shortnames(ffuf):
     in_scope_only = True
 
     async def setup(self):
+        self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist = self.config.get("wordlist", "")
         if not wordlist:

bbot/modules/gowitness.py

@@ -56,7 +56,7 @@ async def setup(self):
         self.threads = self.config.get("threads", 0)
         if not self.threads:
             self.threads = default_thread_count
-        self.proxy = self.scan.config.get("http_proxy", "")
+        self.proxy = self.scan.web_config.get("http_proxy", "")
         self.resolution_x = self.config.get("resolution_x")
         self.resolution_y = self.config.get("resolution_y")
         self.visit_social = self.config.get("social", True)

bbot/modules/report/asn.py

@@ -18,6 +18,7 @@ class asn(BaseReportModule):
     async def setup(self):
         self.asn_counts = {}
         self.asn_cache = {}
+        self.ripe_cache = {}
         self.sources = ["bgpview", "ripe"]
         self.unknown_asn = {
             "asn": "UNKNOWN",
@@ -144,38 +145,42 @@ async def get_asn_ripe(self, ip):
         return asns
 
     async def get_asn_metadata_ripe(self, asn_number):
-        metadata_keys = {
-            "name": ["ASName", "OrgId"],
-            "description": ["OrgName", "OrgTechName", "RTechName"],
-            "country": ["Country"],
-        }
-        url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
-        response = await self.get_url(url, "ASN Metadata", cache=True)
-        if response == False:
-            return False
-        data = response.get("data", {})
-        if not data:
-            data = {}
-        records = data.get("records", [])
-        if not records:
-            records = []
-        emails = set()
-        asn = {k: "" for k in metadata_keys.keys()}
-        for record in records:
-            for item in record:
-                key = item.get("key", "")
-                value = item.get("value", "")
-                for email in await self.helpers.re.extract_emails(value):
-                    emails.add(email.lower())
-                if not key:
-                    continue
-                if value:
-                    for keyname, keyvals in metadata_keys.items():
-                        if key in keyvals and not asn.get(keyname, ""):
-                            asn[keyname] = value
-        asn["emails"] = list(emails)
-        asn["asn"] = str(asn_number)
-        return asn
+        try:
+            return self.ripe_cache[asn_number]
+        except KeyError:
+            metadata_keys = {
+                "name": ["ASName", "OrgId"],
+                "description": ["OrgName", "OrgTechName", "RTechName"],
+                "country": ["Country"],
+            }
+            url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
+            response = await self.get_url(url, "ASN Metadata", cache=True)
+            if response == False:
+                return False
+            data = response.get("data", {})
+            if not data:
+                data = {}
+            records = data.get("records", [])
+            if not records:
+                records = []
+            emails = set()
+            asn = {k: "" for k in metadata_keys.keys()}
+            for record in records:
+                for item in record:
+                    key = item.get("key", "")
+                    value = item.get("value", "")
+                    for email in await self.helpers.re.extract_emails(value):
+                        emails.add(email.lower())
+                    if not key:
+                        continue
+                    if value:
+                        for keyname, keyvals in metadata_keys.items():
+                            if key in keyvals and not asn.get(keyname, ""):
+                                asn[keyname] = value
+            asn["emails"] = list(emails)
+            asn["asn"] = str(asn_number)
+            self.ripe_cache[asn_number] = asn
+            return asn
 
     async def get_asn_bgpview(self, ip):
         url = f"https://api.bgpview.io/ip/{ip}"

bbot/modules/unstructured.py

@@ -67,7 +67,7 @@ class unstructured(BaseModule):
     }
 
     deps_apt = ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"]
-    deps_pip = ["unstructured[all-docs]>=0.5.15,<1.0"]
+    deps_pip = ["unstructured[all-docs]>=0.15.7,<1.0", "nltk>=3.9.0,<4.0"]
 
     scope_distance_modifier = 1
 

bbot/modules/wpscan.py

@@ -62,7 +62,7 @@ async def setup(self):
         self.ignore_events = ["xmlrpc", "readme"]
         self.api_key = self.config.get("api_key", "")
         self.enumerate = self.config.get("enumerate", "vp,vt,tt,cb,dbe,u,m")
-        self.proxy = self.scan.config.get("http_proxy", "")
+        self.proxy = self.scan.web_config.get("http_proxy", "")
         self.threads = self.config.get("threads", 5)
         self.request_timeout = self.config.get("request_timeout", 60)
         self.connection_timeout = self.config.get("connection_timeout", 30)

bbot/scanner/preset/environ.py

@@ -42,7 +42,7 @@ def add_to_path(v, k="PATH", environ=None):
         if _ != v and _ not in deduped_var_list:
             deduped_var_list.append(_)
     deduped_var_list = [v] + deduped_var_list
-    new_var_str = ":".join(deduped_var_list)
+    new_var_str = ":".join(deduped_var_list).strip(":")
     environ[k] = new_var_str
 
 
@@ -107,7 +107,7 @@ def prepare(self):
         environ.update(bbot_environ)
 
         # handle HTTP proxy
-        http_proxy = self.preset.config.get("http_proxy", "")
+        http_proxy = self.preset.config.get("web", {}).get("http_proxy", "")
         if http_proxy:
             environ["HTTP_PROXY"] = http_proxy
             environ["HTTPS_PROXY"] = http_proxy

bbot/scanner/preset/preset.py

@@ -74,7 +74,7 @@ class Preset:
                 "1.2.3.0/24",
                 flags=["subdomain-enum"],
                 modules=["nuclei"],
-                config={"http_proxy": "http://127.0.0.1"}
+                config={"web": {"http_proxy": "http://127.0.0.1"}}
             )
         >>> scan = Scanner(preset=preset)