Dev --> Stable 1.1.2

bbot/cli.py

@@ -5,8 +5,8 @@
 import sys
 import asyncio
 import logging
-import threading
 import traceback
+from aioconsole import ainput
 from omegaconf import OmegaConf
 from contextlib import suppress
 
@@ -306,36 +306,42 @@ async def _main():
                         log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}")
                         input()
 
-                    def keyboard_listen():
-                        allowed_errors = 10
+                    def handle_keyboard_input(keyboard_input):
                         kill_regex = re.compile(r"kill (?P<module>[a-z0-9_]+)")
+                        if keyboard_input:
+                            log.verbose(f'Got keyboard input: "{keyboard_input}"')
+                            kill_match = kill_regex.match(keyboard_input)
+                            if kill_match:
+                                module = kill_match.group("module")
+                                if module in scanner.modules:
+                                    log.hugewarning(f'Killing module: "{module}"')
+                                    scanner.manager.kill_module(module, message="killed by user")
+                                else:
+                                    log.warning(f'Invalid module: "{module}"')
+                        else:
+                            toggle_log_level(logger=log)
+                            scanner.manager.modules_status(_log=True)
+
+                    async def akeyboard_listen():
+                        allowed_errors = 10
                         while 1:
                             keyboard_input = "a"
                             try:
-                                keyboard_input = input()
-                                allowed_errors = 10
+                                keyboard_input = await ainput()
                             except Exception:
                                 allowed_errors -= 1
-                            if keyboard_input:
-                                log.verbose(f'Got keyboard input: "{keyboard_input}"')
-                                kill_match = kill_regex.match(keyboard_input)
-                                if kill_match:
-                                    module = kill_match.group("module")
-                                    if module in scanner.modules:
-                                        log.hugewarning(f'Killing module: "{module}"')
-                                        scanner.manager.kill_module(module, message="killed by user")
-                                    else:
-                                        log.warning(f'Invalid module: "{module}"')
-                            else:
-                                toggle_log_level(logger=log)
-                                scanner.manager.modules_status(_log=True)
+                            handle_keyboard_input(keyboard_input)
                             if allowed_errors <= 0:
                                 break
 
-                    keyboard_listen_thread = threading.Thread(target=keyboard_listen, daemon=True)
-                    keyboard_listen_thread.start()
+                    try:
+                        keyboard_listen_task = asyncio.create_task(akeyboard_listen())
 
-                    await scanner.async_start_without_generator()
+                        await scanner.async_start_without_generator()
+                    finally:
+                        keyboard_listen_task.cancel()
+                        with suppress(asyncio.CancelledError):
+                            await keyboard_listen_task
 
             except bbot.core.errors.ScanError as e:
                 log_to_stderr(str(e), level="ERROR")

bbot/core/configurator/__init__.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from pathlib import Path
 from omegaconf import OmegaConf
@@ -41,20 +42,28 @@
 sentinel = object()
 
 
+exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$")
+
+
 def check_cli_args():
-    for c in args.cli_config:
-        if not is_file(c):
-            c = c.split("=")[0].strip()
-            v = OmegaConf.select(default_config, c, default=sentinel)
-            if v is sentinel:
+    conf = [a for a in args.cli_config if not is_file(a)]
+    all_options = None
+    for c in conf:
+        c = c.split("=")[0].strip()
+        v = OmegaConf.select(default_config, c, default=sentinel)
+        # if option isn't in the default config
+        if v is sentinel:
+            if exclude_from_validation.match(c):
+                continue
+            if all_options is None:
                 from ...modules import module_loader
 
                 modules_options = set()
                 for module_options in module_loader.modules_options().values():
                     modules_options.update(set(o[0] for o in module_options))
                 global_options = set(default_config.keys()) - {"modules", "output_modules"}
                 all_options = global_options.union(modules_options)
-                match_and_exit(c, all_options, msg="module option")
+            match_and_exit(c, all_options, msg="module option")
 
 
 def ensure_config_files():

bbot/core/event/base.py

@@ -1129,6 +1129,18 @@ class GEOLOCATION(BaseEvent):
     _always_emit = True
 
 
+class PASSWORD(BaseEvent):
+    _always_emit = True
+
+
+class HASHED_PASSWORD(BaseEvent):
+    _always_emit = True
+
+
+class USERNAME(BaseEvent):
+    _always_emit = True
+
+
 class SOCIAL(DictEvent):
     _always_emit = True
 

bbot/core/helpers/cloud/azure.py

@@ -2,25 +2,47 @@
 
 
 class Azure(BaseCloudProvider):
+    # mostly pulled from https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure
     domains = [
+        "azconfig.io",
         "azmk8s.io",
         "azure-api.net",
+        "azure-api.us",
+        "azure-automation.net",
+        "azure-automation.us",
+        "azure-devices.net",
+        "azure-devices.us",
         "azure-mobile.net",
         "azure.com",
         "azure.net",
+        "azure.us",
         "azurecontainer.io",
         "azurecr.io",
+        "azurecr.us",
         "azuredatalakestore.net",
         "azureedge.net",
         "azurefd.net",
         "azurehdinsight.net",
+        "azurehdinsight.us",
         "azurewebsites.net",
+        "botframework.com",
         "cloudapp.net",
-        "windows.net",
+        "loganalytics.io",
+        "loganalytics.us",
+        "microsoft.us",
+        "microsoftonline.com",
+        "microsoftonline.us",
         "onmicrosoft.com",
+        "powerbi.com",
+        "powerbigov.us",
         "trafficmanager.net",
+        "usgovcloudapi.net",
+        "usgovtrafficmanager.net",
         "visualstudio.com",
         "vo.msecnd.net",
+        "windows.net",
+        "windowsazure.com",
+        "windowsazure.us",
     ]
 
     bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]"

bbot/core/helpers/regexes.py

@@ -31,7 +31,7 @@
 _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"
 hostname_regex = re.compile(r"^" + _hostname_regex + r"$", re.I)
 
-_email_regex = r"(?:[^\W_][\w\-\.\+]{,100})@" + _dns_name_regex
+_email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
 email_regex = re.compile(_email_regex, re.I)
 
 _ptr_regex = r"(?:[0-9]{1,3}[-_\.]){3}[0-9]{1,3}"

bbot/core/helpers/web.py

@@ -7,6 +7,7 @@
 import traceback
 from pathlib import Path
 from bs4 import BeautifulSoup
+from contextlib import asynccontextmanager
 
 from httpx._models import Cookies
 
@@ -131,7 +132,10 @@ class WebHelper:
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.http_debug = self.parent_helper.config.get("http_debug", False)
+        self._ssl_context_noverify = None
         self.ssl_verify = self.parent_helper.config.get("ssl_verify", False)
+        if self.ssl_verify is False:
+            self.ssl_verify = self.ssl_context_noverify()
         self.web_client = self.AsyncClient(persist_cookies=False)
 
     def AsyncClient(self, *args, **kwargs):
@@ -216,7 +220,7 @@ async def request(self, *args, **kwargs):
         if client_kwargs:
             client = self.AsyncClient(**client_kwargs)
 
-        try:
+        async with self._acatch(url, raise_error):
             if self.http_debug:
                 logstr = f"Web request: {str(args)}, {str(kwargs)}"
                 log.debug(logstr)
@@ -226,41 +230,6 @@ async def request(self, *args, **kwargs):
                     f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
                 )
             return response
-        except httpx.PoolTimeout:
-            # this block exists because of this:
-            #  https://github.com/encode/httpcore/discussions/783
-            log.verbose(f"PoolTimeout to URL: {url}")
-            self.web_client = self.AsyncClient(persist_cookies=False)
-            return await self.request(*args, **kwargs)
-        except httpx.TimeoutException:
-            log.verbose(f"HTTP timeout to URL: {url}")
-            if raise_error:
-                raise
-        except httpx.ConnectError:
-            log.verbose(f"HTTP connect failed to URL: {url}")
-            if raise_error:
-                raise
-        except httpx.RequestError as e:
-            log.trace(f"Error with request to URL: {url}: {e}")
-            log.trace(traceback.format_exc())
-            if raise_error:
-                raise
-        except ssl.SSLError as e:
-            msg = f"SSL error with request to URL: {url}: {e}"
-            log.trace(msg)
-            log.trace(traceback.format_exc())
-            if raise_error:
-                raise httpx.RequestError(msg)
-        except anyio.EndOfStream as e:
-            msg = f"AnyIO error with request to URL: {url}: {e}"
-            log.trace(msg)
-            log.trace(traceback.format_exc())
-            if raise_error:
-                raise httpx.RequestError(msg)
-        except BaseException as e:
-            log.trace(f"Unhandled exception with request to URL: {url}: {e}")
-            log.trace(traceback.format_exc())
-            raise
 
     async def download(self, url, **kwargs):
         """
@@ -272,9 +241,11 @@ async def download(self, url, **kwargs):
             url (str): The URL of the file to download.
             filename (str, optional): The filename to save the downloaded file as.
                 If not provided, will generate based on URL.
+            max_size (str or int): Maximum filesize as a string ("5MB") or integer in bytes.
             cache_hrs (float, optional): The number of hours to cache the downloaded file.
                 A negative value disables caching. Defaults to -1.
             method (str, optional): The HTTP method to use for the request, defaults to 'GET'.
+            raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.
             **kwargs: Additional keyword arguments to pass to the httpx request.
 
         Returns:
@@ -285,28 +256,48 @@ async def download(self, url, **kwargs):
         """
         success = False
         filename = kwargs.pop("filename", self.parent_helper.cache_filename(url))
+        follow_redirects = kwargs.pop("follow_redirects", True)
+        max_size = kwargs.pop("max_size", None)
+        warn = kwargs.pop("warn", True)
+        raise_error = kwargs.pop("raise_error", False)
+        if max_size is not None:
+            max_size = self.parent_helper.human_to_bytes(max_size)
         cache_hrs = float(kwargs.pop("cache_hrs", -1))
+        total_size = 0
+        chunk_size = 8192
         log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}")
         if cache_hrs > 0 and self.parent_helper.is_cached(url):
             log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}")
             success = True
         else:
             # kwargs["raise_error"] = True
             # kwargs["stream"] = True
+            kwargs["follow_redirects"] = follow_redirects
             if not "method" in kwargs:
                 kwargs["method"] = "GET"
             try:
-                async with self.AsyncClient().stream(url=url, **kwargs) as response:
+                async with self._acatch(url, raise_error), self.AsyncClient().stream(url=url, **kwargs) as response:
                     status_code = getattr(response, "status_code", 0)
                     log.debug(f"Download result: HTTP {status_code}")
                     if status_code != 0:
                         response.raise_for_status()
                         with open(filename, "wb") as f:
-                            async for chunk in response.aiter_bytes(chunk_size=8192):
+                            agen = response.aiter_bytes(chunk_size=chunk_size)
+                            async for chunk in agen:
+                                if max_size is not None and total_size + chunk_size > max_size:
+                                    log.verbose(
+                                        f"Filesize of {url} exceeds {self.parent_helper.bytes_to_human(max_size)}, file will be truncated"
+                                    )
+                                    agen.aclose()
+                                    break
+                                total_size += chunk_size
                                 f.write(chunk)
                         success = True
             except httpx.HTTPError as e:
-                log.warning(f"Failed to download {url}: {e}")
+                log_fn = log.verbose
+                if warn:
+                    log_fn = log.warning
+                log_fn(f"Failed to download {url}: {e}")
                 return
 
         if success:
@@ -465,7 +456,7 @@ async def curl(self, *args, **kwargs):
             curl_command.append("--path-as-is")
 
         # respect global ssl verify settings
-        if self.ssl_verify == False:
+        if self.ssl_verify is not True:
             curl_command.append("-k")
 
         headers = kwargs.get("headers", {})
@@ -574,6 +565,61 @@ def is_spider_danger(self, source_event, url):
             return True
         return False
 
+    def ssl_context_noverify(self):
+        if self._ssl_context_noverify is None:
+            ssl_context = ssl.create_default_context()
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+            ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3
+            ssl_context.set_ciphers("ALL:@SECLEVEL=0")
+            ssl_context.options |= 0x4  # Add the OP_LEGACY_SERVER_CONNECT option
+            self._ssl_context_noverify = ssl_context
+        return self._ssl_context_noverify
+
+    @asynccontextmanager
+    async def _acatch(self, url, raise_error):
+        """
+        Asynchronous context manager to handle various httpx errors during a request.
+
+        Yields:
+            None
+
+        Note:
+            This function is internal and should generally not be used directly.
+            `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function.
+        """
+        try:
+            yield
+        except httpx.TimeoutException:
+            log.verbose(f"HTTP timeout to URL: {url}")
+            if raise_error:
+                raise
+        except httpx.ConnectError:
+            log.verbose(f"HTTP connect failed to URL: {url}")
+            if raise_error:
+                raise
+        except httpx.RequestError as e:
+            log.trace(f"Error with request to URL: {url}: {e}")
+            log.trace(traceback.format_exc())
+            if raise_error:
+                raise
+        except ssl.SSLError as e:
+            msg = f"SSL error with request to URL: {url}: {e}"
+            log.trace(msg)
+            log.trace(traceback.format_exc())
+            if raise_error:
+                raise httpx.RequestError(msg)
+        except anyio.EndOfStream as e:
+            msg = f"AnyIO error with request to URL: {url}: {e}"
+            log.trace(msg)
+            log.trace(traceback.format_exc())
+            if raise_error:
+                raise httpx.RequestError(msg)
+        except BaseException as e:
+            log.trace(f"Unhandled exception with request to URL: {url}: {e}")
+            log.trace(traceback.format_exc())
+            raise
+
 
 user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
 pass_keywords = [re.compile(r, re.I) for r in ["pass"]]