Merge pull request #1875 from blacklanternsecurity/fast-mode

--fast mode

.github/workflows/docs_updater.yml

@@ -30,5 +30,5 @@ jobs:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           branch: update-docs
           base: dev
-          title: "Daily Docs Update"
+          title: "Automated Docs Update"
           body: "This is an automated pull request to update the documentation."

bbot/core/helpers/depsinstaller/installer.py

@@ -14,7 +14,7 @@
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform, rm_at_exit
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -176,10 +176,13 @@ async def pip_install(self, packages, constraints=None):
 
         command = [sys.executable, "-m", "pip", "install", "--upgrade"] + packages
 
-        if constraints:
-            constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
-            command.append("--constraint")
-            command.append(constraints_tempfile)
+        # if no custom constraints are provided, use the constraints of the currently installed version of bbot
+        if constraints is not None:
+            constraints = get_python_constraints()
+
+        constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
+        command.append("--constraint")
+        command.append(constraints_tempfile)
 
         process = None
         try:

bbot/core/helpers/misc.py

@@ -2807,3 +2807,21 @@ def safe_format(s, **kwargs):
     Format string while ignoring unused keys (prevents KeyError)
     """
     return s.format_map(SafeDict(kwargs))
+
+
+def get_python_constraints():
+    req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE)
+
+    def clean_requirement(req_string):
+        # Extract package name and version constraints from format like "package (>=1.0,<2.0)"
+        match = req_regex.match(req_string)
+        if match:
+            name, constraints = match.groups()
+            return f"{name.strip()}{constraints}"
+
+        return req_string
+
+    from importlib.metadata import distribution
+
+    dist = distribution("bbot")
+    return [clean_requirement(r) for r in dist.requires]

bbot/defaults.yml

@@ -14,6 +14,9 @@ folder_blobs: false
 ### SCOPE ###
 
 scope:
+  # strict scope means only exact DNS names are considered in-scope
+  # subdomains are not included unless they are explicitly provided in the target list
+  strict: false
   # Filter by scope distance which events are displayed in the output
   # 0 == show only in-scope events (affiliates are always shown)
   # 1 == show all events up to distance-1 (1 hop from target)

bbot/modules/internal/speculate.py

@@ -32,10 +32,11 @@ class speculate(BaseInternalModule):
         "author": "@liquidsec",
     }
 
-    options = {"max_hosts": 65536, "ports": "80,443"}
+    options = {"max_hosts": 65536, "ports": "80,443", "essential_only": False}
     options_desc = {
         "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events",
         "ports": "The set of ports to speculate on",
+        "essential_only": "Only enable essential speculate features (no extra discovery)",
     }
     scope_distance_modifier = 1
     _priority = 4
@@ -52,6 +53,7 @@ async def setup(self):
         self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
         self.range_to_ip = True
         self.dns_disable = self.scan.config.get("dns", {}).get("disable", False)
+        self.essential_only = self.config.get("essential_only", False)
         self.org_stubs_seen = set()
 
         port_string = self.config.get("ports", "80,443")
@@ -75,6 +77,14 @@ async def setup(self):
         return True
 
     async def handle_event(self, event):
+        ### BEGIN ESSENTIAL SPECULATION ###
+        # These features are required for smooth operation of bbot
+        # I.e. they are not "osinty" or intended to discover anything, they only compliment other modules
+
+        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
+        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
+        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
+
         # generate individual IP addresses from IP range
         if event.type == "IP_RANGE" and self.range_to_ip:
             net = ipaddress.ip_network(event.data)
@@ -89,6 +99,28 @@ async def handle_event(self, event):
                     context=f"speculate converted range into individual IP_ADDRESS: {ip}",
                 )
 
+        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
+        if speculate_open_ports:
+            # don't act on unresolved DNS_NAMEs
+            usable_dns = False
+            if event.type == "DNS_NAME":
+                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
+                    usable_dns = True
+
+            if event.type == "IP_ADDRESS" or usable_dns:
+                for port in self.ports:
+                    await self.emit_event(
+                        self.helpers.make_netloc(event.data, port),
+                        "OPEN_TCP_PORT",
+                        parent=event,
+                        internal=True,
+                        context="speculated {event.type}: {event.data}",
+                    )
+
+        ### END ESSENTIAL SPECULATION ###
+        if self.essential_only:
+            return
+
         # parent domains
         if event.type.startswith("DNS_NAME"):
             parent = self.helpers.parent_domain(event.host_original)
@@ -97,10 +129,6 @@ async def handle_event(self, event):
                     parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}"
                 )
 
-        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
-        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
-        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
-
         # URL --> OPEN_TCP_PORT
         event_is_url = event.type == "URL"
         if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
@@ -144,24 +172,6 @@ async def handle_event(self, event):
                     context="speculated {event.type}: {event.data}",
                 )
 
-        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
-        if speculate_open_ports:
-            # don't act on unresolved DNS_NAMEs
-            usable_dns = False
-            if event.type == "DNS_NAME":
-                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
-                    usable_dns = True
-
-            if event.type == "IP_ADDRESS" or usable_dns:
-                for port in self.ports:
-                    await self.emit_event(
-                        self.helpers.make_netloc(event.data, port),
-                        "OPEN_TCP_PORT",
-                        parent=event,
-                        internal=True,
-                        context="speculated {event.type}: {event.data}",
-                    )
-
         # ORG_STUB from TLD, SOCIAL, AZURE_TENANT
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0:

bbot/modules/leakix.py

@@ -15,20 +15,19 @@ class leakix(subdomain_enum_apikey):
     }
 
     base_url = "https://leakix.net"
-    ping_url = f"{base_url}/host/1.2.3.4.5"
+    ping_url = f"{base_url}/host/1.1.1.1"
 
     async def setup(self):
         ret = await super(subdomain_enum_apikey, self).setup()
-        self.headers = {"Accept": "application/json"}
         self.api_key = self.config.get("api_key", "")
         if self.api_key:
-            self.headers["api-key"] = self.api_key
             return await self.require_api_key()
         return ret
 
     def prepare_api_request(self, url, kwargs):
         if self.api_key:
             kwargs["headers"]["api-key"] = self.api_key
+            kwargs["headers"]["Accept"] = "application/json"
         return url, kwargs
 
     async def request_url(self, query):

bbot/presets/fast.yml

@@ -0,0 +1,16 @@
+description: Scan only the provided targets as fast as possible - no extra discovery
+
+exclude_modules:
+  - excavate
+
+config:
+  # only scan the exact targets specified
+  scope:
+    strict: true
+  # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc
+  dns:
+    minimal: true
+  # essential speculation only
+  modules:
+    speculate:
+      essential_only: true

bbot/scanner/preset/args.py

@@ -91,7 +91,6 @@ def preset_from_args(self):
             *self.parsed.targets,
             whitelist=self.parsed.whitelist,
             blacklist=self.parsed.blacklist,
-            strict_scope=self.parsed.strict_scope,
             name="args_preset",
         )
 
@@ -149,6 +148,9 @@ def preset_from_args(self):
         if self.parsed.force:
             args_preset.force_start = self.parsed.force
 
+        if self.parsed.proxy:
+            args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
+
         if self.parsed.custom_headers:
             args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
 
@@ -165,6 +167,10 @@ def preset_from_args(self):
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
 
+        # strict scope
+        if self.parsed.strict_scope:
+            args_preset.core.merge_custom({"scope": {"strict": True}})
+
         return args_preset
 
     def create_parser(self, *args, **kwargs):
@@ -265,6 +271,11 @@ def create_parser(self, *args, **kwargs):
             help="Run scan even in the case of condition violations or failed module setups",
         )
         scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
+        scan.add_argument(
+            "--fast-mode",
+            action="store_true",
+            help="Scan only the provided targets as fast as possible, with no extra discovery",
+        )
         scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
         scan.add_argument(
             "--current-preset",
@@ -310,6 +321,7 @@ def create_parser(self, *args, **kwargs):
 
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
+        misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
         misc.add_argument(
             "-H",
             "--custom-headers",
@@ -359,6 +371,10 @@ def sanitize_args(self):
             custom_headers_dict[k] = v
         self.parsed.custom_headers = custom_headers_dict
 
+        # --fast-mode
+        if self.parsed.fast_mode:
+            self.parsed.preset += ["fast"]
+
     def validate(self):
         # validate config options
         sentinel = object()

bbot/scanner/preset/preset.py

@@ -47,7 +47,6 @@ class Preset:
         target (Target): Target(s) of scan.
         whitelist (Target): Scan whitelist (by default this is the same as `target`).
         blacklist (Target): Scan blacklist (this takes ultimate precedence).
-        strict_scope (bool): If True, subdomains of targets are not considered to be in-scope.
         helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.
         output_dir (pathlib.Path): Output directory for scan.
         scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy".
@@ -87,7 +86,6 @@ def __init__(
         *targets,
         whitelist=None,
         blacklist=None,
-        strict_scope=False,
         modules=None,
         output_modules=None,
         exclude_modules=None,
@@ -117,7 +115,6 @@ def __init__(
             *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.
             whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.
             blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
-            strict_scope (bool, optional): If True, subdomains of targets are not in-scope.
             modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
             output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
             exclude_modules (list[str], optional): List of modules to exclude from the scan.
@@ -234,7 +231,6 @@ def __init__(
         self.module_dirs = module_dirs
 
         # target / whitelist / blacklist
-        self.strict_scope = strict_scope
         # these are temporary receptacles until they all get .baked() together
         self._seeds = set(targets if targets else [])
         self._whitelist = set(whitelist) if whitelist else whitelist
@@ -353,7 +349,6 @@ def merge(self, other):
             else:
                 self._whitelist.update(other._whitelist)
         self._blacklist.update(other._blacklist)
-        self.strict_scope = self.strict_scope or other.strict_scope
 
         # module dirs
         self.module_dirs = self.module_dirs.union(other.module_dirs)
@@ -537,6 +532,14 @@ def config(self):
     def web_config(self):
         return self.core.config.get("web", {})
 
+    @property
+    def scope_config(self):
+        return self.config.get("scope", {})
+
+    @property
+    def strict_scope(self):
+        return self.scope_config.get("strict", False)
+
     def apply_log_level(self, apply_core=False):
         # silent takes precedence
         if self.silent:
@@ -635,7 +638,6 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
             debug=preset_dict.get("debug", False),
             silent=preset_dict.get("silent", False),
             config=preset_dict.get("config"),
-            strict_scope=preset_dict.get("strict_scope", False),
             module_dirs=preset_dict.get("module_dirs", []),
             include=list(preset_dict.get("include", [])),
             scan_name=preset_dict.get("scan_name"),
@@ -764,8 +766,6 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False)
                 preset_dict["whitelist"] = whitelist
             if blacklist:
                 preset_dict["blacklist"] = blacklist
-        if self.strict_scope:
-            preset_dict["strict_scope"] = True
 
         # flags + modules
         if self.require_flags:

bbot/test/bbot_fixtures.py

@@ -15,8 +15,8 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
 log = logging.getLogger(f"bbot.test.fixtures")
@@ -229,4 +229,7 @@ def install_all_python_deps():
     deps_pip = set()
     for module in DEFAULT_PRESET.module_loader.preloaded().values():
         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+
+    constraint_file = tempwordlist(get_python_constraints())
+
+    subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))