Dev -> Stable 2.3.0

.github/dependabot.yml

@@ -6,3 +6,11 @@ updates:
       interval: "weekly"
     target-branch: "dev"
     open-pull-requests-limit: 10
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/distro_tests.yml

@@ -24,17 +24,17 @@ jobs:
             if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
               export DEBIAN_FRONTEND=noninteractive
               apt-get update
-              apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
+              apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
             elif [ "$ID" = "alpine" ]; then
-              apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
+              apk add --no-cache bash gcc g++ musl-dev libffi-dev docker curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
             elif [ "$ID" = "arch" ]; then
-              pacman -Syu --noconfirm curl git bash base-devel
+              pacman -Syu --noconfirm curl docker git bash base-devel
             elif [ "$ID" = "fedora" ]; then
-              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
+              dnf install -y curl docker git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
             elif [ "$ID" = "gentoo" ]; then
               echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
               emerge-webrsync
-              emerge --update --newuse dev-vcs/git media-libs/mesa curl bash
+              emerge --update --newuse dev-vcs/git media-libs/mesa curl docker bash
             fi
           fi
 

.github/workflows/docs_updater.yml

@@ -30,5 +30,5 @@ jobs:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           branch: update-docs
           base: dev
-          title: "Daily Docs Update"
+          title: "Automated Docs Update"
           body: "This is an automated pull request to update the documentation."

.github/workflows/tests.yml

@@ -48,7 +48,7 @@ jobs:
           poetry install
       - name: Run tests
         run: |
-          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         uses: actions/upload-artifact@v3
         with:

README.md

@@ -91,6 +91,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row
@@ -191,10 +195,10 @@ flags:
 
 ```bash
 # everything everywhere all at once
-bbot -t evilcorp.com -p kitchen-sink
+bbot -t evilcorp.com -p kitchen-sink --allow-deadly
 
 # roughly equivalent to:
-bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots
+bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly
 ```
 
 <!-- BBOT KITCHEN-SINK PRESET EXPANDABLE -->
@@ -236,6 +240,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

bbot/cli.py

@@ -174,7 +174,7 @@ async def _main():
             if sys.stdin.isatty():
 
                 # warn if any targets belong directly to a cloud provider
-                for event in scan.target.events:
+                for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":
                         cloudcheck_result = scan.helpers.cloudcheck(event.host)
                         if cloudcheck_result:

bbot/core/config/logger.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import atexit
 import logging
@@ -9,6 +10,7 @@
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
+from ..multiprocess import SHARED_INTERPRETER_STATE
 
 
 debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
@@ -65,8 +67,9 @@ def __init__(self, core):
 
         self.listener = None
 
-        self.process_name = multiprocessing.current_process().name
-        if self.process_name == "MainProcess":
+        # if we haven't set up logging yet, do it now
+        if not "_BBOT_LOGGING_SETUP" in os.environ:
+            os.environ["_BBOT_LOGGING_SETUP"] = "1"
             self.queue = multiprocessing.Queue()
             self.setup_queue_handler()
             # Start the QueueListener
@@ -113,7 +116,7 @@ def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
 
         self.core_logger.setLevel(log_level)
         # disable asyncio logging for child processes
-        if self.process_name != "MainProcess":
+        if not SHARED_INTERPRETER_STATE.is_main_process:
             logging.getLogger("asyncio").setLevel(logging.ERROR)
 
     def addLoggingLevel(self, levelName, levelNum, methodName=None):

bbot/core/core.py

@@ -6,6 +6,7 @@
 from omegaconf import OmegaConf
 
 from bbot.errors import BBOTError
+from .multiprocess import SHARED_INTERPRETER_STATE
 
 
 DEFAULT_CONFIG = None
@@ -41,9 +42,23 @@ def __init__(self):
         self.logger
         self.log = logging.getLogger("bbot.core")
 
+        self._prep_multiprocessing()
+
+    def _prep_multiprocessing(self):
         import multiprocessing
+        from .helpers.process import BBOTProcess
+
+        if SHARED_INTERPRETER_STATE.is_main_process:
+            # if this is the main bbot process, set the logger and queue for the first time
+            from functools import partialmethod
 
-        self.process_name = multiprocessing.current_process().name
+            BBOTProcess.__init__ = partialmethod(
+                BBOTProcess.__init__, log_level=self.logger.log_level, log_queue=self.logger.queue
+            )
+
+        # this makes our process class the default for process pools, etc.
+        mp_context = multiprocessing.get_context("spawn")
+        mp_context.Process = BBOTProcess
 
     @property
     def home(self):
@@ -187,12 +202,14 @@ def create_process(self, *args, **kwargs):
         if os.environ.get("BBOT_TESTING", "") == "True":
             process = self.create_thread(*args, **kwargs)
         else:
-            if self.process_name == "MainProcess":
+            if SHARED_INTERPRETER_STATE.is_scan_process:
                 from .helpers.process import BBOTProcess
 
                 process = BBOTProcess(*args, **kwargs)
             else:
-                raise BBOTError(f"Tried to start server from process {self.process_name}")
+                import multiprocessing
+
+                raise BBOTError(f"Tried to start server from process {multiprocessing.current_process().name}")
         process.daemon = True
         return process
 

bbot/core/engine.py

@@ -10,13 +10,15 @@
 import contextlib
 import contextvars
 import zmq.asyncio
+import multiprocessing
 from pathlib import Path
 from concurrent.futures import CancelledError
 from contextlib import asynccontextmanager, suppress
 
 from bbot.core import CORE
 from bbot.errors import BBOTEngineError
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
 from bbot.core.helpers.misc import rand_string, in_exception_chain
 
 
@@ -264,10 +266,8 @@ def available_commands(self):
         return [s for s in self.CMDS if isinstance(s, str)]
 
     def start_server(self):
-        import multiprocessing
-
         process_name = multiprocessing.current_process().name
-        if process_name == "MainProcess":
+        if SHARED_INTERPRETER_STATE.is_scan_process:
             kwargs = dict(self.server_kwargs)
             # if we're in tests, we use a single event loop to avoid weird race conditions
             # this allows us to more easily mock http, etc.
@@ -641,7 +641,7 @@ async def finished_tasks(self, tasks, timeout=None):
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):
                     self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})")
-                    for task in tasks:
+                    for task in list(tasks):
                         task.cancel()
                         self._await_cancelled_task(task)
                 else:
@@ -683,5 +683,5 @@ async def cancel_all_tasks(self):
         for client_id in list(self.tasks):
             await self.cancel_task(client_id)
         for client_id, tasks in self.child_tasks.items():
-            for task in tasks:
+            for task in list(tasks):
                 await self._await_cancelled_task(task)

bbot/core/event/base.py

@@ -341,6 +341,21 @@ def host_original(self):
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -1003,13 +1018,15 @@ def __init__(self, *args, **kwargs):
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -1112,8 +1129,7 @@ def __init__(self, *args, **kwargs):
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1559,6 +1575,8 @@ def __init__(self, *args, **kwargs):
                 self.add_tag("compressed")
                 self.add_tag(f"{compression}-archive")
                 self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
 
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
@@ -1639,23 +1657,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
+            event.discovery_context = context
         if internal == True:
-            data.internal = True
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.tags = tags.union(event.tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)
@@ -1685,6 +1703,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 

bbot/core/helpers/bloom.py

@@ -64,8 +64,15 @@ def _fnv1a_hash(self, data):
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

bbot/core/helpers/command.py

@@ -210,9 +210,10 @@ async def _write_proc_line(proc, chunk):
         return True
     except Exception as e:
         proc_args = [str(s) for s in getattr(proc, "args", [])]
-        command = " ".join(proc_args)
-        log.warning(f"Error writing line to stdin for command: {command}: {e}")
-        log.trace(traceback.format_exc())
+        command = " ".join(proc_args).strip()
+        if command:
+            log.warning(f"Error writing line to stdin for command: {command}: {e}")
+            log.trace(traceback.format_exc())
         return False