Merge pull request #777 from blacklanternsecurity/dev

Dev --> Stable

bbot/cli.py

@@ -5,8 +5,8 @@
 import sys
 import asyncio
 import logging
-import threading
 import traceback
+from aioconsole import ainput
 from omegaconf import OmegaConf
 from contextlib import suppress
 
@@ -306,36 +306,42 @@ async def _main():
                         log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}")
                         input()
 
-                    def keyboard_listen():
-                        allowed_errors = 10
+                    def handle_keyboard_input(keyboard_input):
                         kill_regex = re.compile(r"kill (?P<module>[a-z0-9_]+)")
+                        if keyboard_input:
+                            log.verbose(f'Got keyboard input: "{keyboard_input}"')
+                            kill_match = kill_regex.match(keyboard_input)
+                            if kill_match:
+                                module = kill_match.group("module")
+                                if module in scanner.modules:
+                                    log.hugewarning(f'Killing module: "{module}"')
+                                    scanner.manager.kill_module(module, message="killed by user")
+                                else:
+                                    log.warning(f'Invalid module: "{module}"')
+                        else:
+                            toggle_log_level(logger=log)
+                            scanner.manager.modules_status(_log=True)
+
+                    async def akeyboard_listen():
+                        allowed_errors = 10
                         while 1:
                             keyboard_input = "a"
                             try:
-                                keyboard_input = input()
-                                allowed_errors = 10
+                                keyboard_input = await ainput()
                             except Exception:
                                 allowed_errors -= 1
-                            if keyboard_input:
-                                log.verbose(f'Got keyboard input: "{keyboard_input}"')
-                                kill_match = kill_regex.match(keyboard_input)
-                                if kill_match:
-                                    module = kill_match.group("module")
-                                    if module in scanner.modules:
-                                        log.hugewarning(f'Killing module: "{module}"')
-                                        scanner.manager.kill_module(module, message="killed by user")
-                                    else:
-                                        log.warning(f'Invalid module: "{module}"')
-                            else:
-                                toggle_log_level(logger=log)
-                                scanner.manager.modules_status(_log=True)
+                            handle_keyboard_input(keyboard_input)
                             if allowed_errors <= 0:
                                 break
 
-                    keyboard_listen_thread = threading.Thread(target=keyboard_listen, daemon=True)
-                    keyboard_listen_thread.start()
+                    try:
+                        keyboard_listen_task = asyncio.create_task(akeyboard_listen())
 
-                    await scanner.async_start_without_generator()
+                        await scanner.async_start_without_generator()
+                    finally:
+                        keyboard_listen_task.cancel()
+                        with suppress(asyncio.CancelledError):
+                            await keyboard_listen_task
 
             except bbot.core.errors.ScanError as e:
                 log_to_stderr(str(e), level="ERROR")

bbot/core/configurator/__init__.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from pathlib import Path
 from omegaconf import OmegaConf
@@ -41,20 +42,28 @@
 sentinel = object()
 
 
+exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$")
+
+
 def check_cli_args():
-    for c in args.cli_config:
-        if not is_file(c):
-            c = c.split("=")[0].strip()
-            v = OmegaConf.select(default_config, c, default=sentinel)
-            if v is sentinel:
+    conf = [a for a in args.cli_config if not is_file(a)]
+    all_options = None
+    for c in conf:
+        c = c.split("=")[0].strip()
+        v = OmegaConf.select(default_config, c, default=sentinel)
+        # if option isn't in the default config
+        if v is sentinel:
+            if exclude_from_validation.match(c):
+                continue
+            if all_options is None:
                 from ...modules import module_loader
 
                 modules_options = set()
                 for module_options in module_loader.modules_options().values():
                     modules_options.update(set(o[0] for o in module_options))
                 global_options = set(default_config.keys()) - {"modules", "output_modules"}
                 all_options = global_options.union(modules_options)
-                match_and_exit(c, all_options, msg="module option")
+            match_and_exit(c, all_options, msg="module option")
 
 
 def ensure_config_files():

bbot/core/event/base.py

@@ -1129,6 +1129,18 @@ class GEOLOCATION(BaseEvent):
     _always_emit = True
 
 
+class PASSWORD(BaseEvent):
+    _always_emit = True
+
+
+class HASHED_PASSWORD(BaseEvent):
+    _always_emit = True
+
+
+class USERNAME(BaseEvent):
+    _always_emit = True
+
+
 class SOCIAL(DictEvent):
     _always_emit = True
 

bbot/core/helpers/cloud/azure.py

@@ -2,25 +2,47 @@
 
 
 class Azure(BaseCloudProvider):
+    # mostly pulled from https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure
     domains = [
+        "azconfig.io",
         "azmk8s.io",
         "azure-api.net",
+        "azure-api.us",
+        "azure-automation.net",
+        "azure-automation.us",
+        "azure-devices.net",
+        "azure-devices.us",
         "azure-mobile.net",
         "azure.com",
         "azure.net",
+        "azure.us",
         "azurecontainer.io",
         "azurecr.io",
+        "azurecr.us",
         "azuredatalakestore.net",
         "azureedge.net",
         "azurefd.net",
         "azurehdinsight.net",
+        "azurehdinsight.us",
         "azurewebsites.net",
+        "botframework.com",
         "cloudapp.net",
-        "windows.net",
+        "loganalytics.io",
+        "loganalytics.us",
+        "microsoft.us",
+        "microsoftonline.com",
+        "microsoftonline.us",
         "onmicrosoft.com",
+        "powerbi.com",
+        "powerbigov.us",
         "trafficmanager.net",
+        "usgovcloudapi.net",
+        "usgovtrafficmanager.net",
         "visualstudio.com",
         "vo.msecnd.net",
+        "windows.net",
+        "windowsazure.com",
+        "windowsazure.us",
     ]
 
     bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]"

bbot/core/helpers/regexes.py

@@ -31,7 +31,7 @@
 _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"
 hostname_regex = re.compile(r"^" + _hostname_regex + r"$", re.I)
 
-_email_regex = r"(?:[^\W_][\w\-\.\+]{,100})@" + _dns_name_regex
+_email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
 email_regex = re.compile(_email_regex, re.I)
 
 _ptr_regex = r"(?:[0-9]{1,3}[-_\.]){3}[0-9]{1,3}"

bbot/core/helpers/validators.py

@@ -221,6 +221,7 @@ def collapse_urls(urls, threshold=10):
         ["http://evilcorp.com/user/11111/info"]
 
     """
+    log.verbose(f"Collapsing {len(urls):,} URLs")
     url_hashes = {}
     for url in urls:
         try: