Merge branch 'dev' into python-3.13

.github/dependabot.yml

@@ -6,3 +6,11 @@ updates:
       interval: "weekly"
     target-branch: "dev"
     open-pull-requests-limit: 10
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/distro_tests.yml

@@ -24,17 +24,17 @@ jobs:
             if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
               export DEBIAN_FRONTEND=noninteractive
               apt-get update
-              apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
+              apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
             elif [ "$ID" = "alpine" ]; then
-              apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
+              apk add --no-cache bash gcc g++ musl-dev libffi-dev docker curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
             elif [ "$ID" = "arch" ]; then
-              pacman -Syu --noconfirm curl git bash base-devel
+              pacman -Syu --noconfirm curl docker git bash base-devel
             elif [ "$ID" = "fedora" ]; then
-              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
+              dnf install -y curl docker git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
             elif [ "$ID" = "gentoo" ]; then
               echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
               emerge-webrsync
-              emerge --update --newuse dev-vcs/git media-libs/mesa curl bash
+              emerge --update --newuse dev-vcs/git media-libs/mesa curl docker bash
             fi
           fi
 

.github/workflows/tests.yml

@@ -15,19 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: psf/black@stable
-        with:
-          options: "--check"
-      - name: Install Python 3
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          pip install flake8
-      - name: flake8
-        run: |
-          flake8
+      - run: |
+          pipx install ruff
+          ruff check
+          ruff format
   test:
     needs: lint
     runs-on: ubuntu-latest

README.md

@@ -91,6 +91,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row
@@ -191,10 +195,10 @@ flags:
 
 ```bash
 # everything everywhere all at once
-bbot -t evilcorp.com -p kitchen-sink
+bbot -t evilcorp.com -p kitchen-sink --allow-deadly
 
 # roughly equivalent to:
-bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots
+bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly
 ```
 
 <!-- BBOT KITCHEN-SINK PRESET EXPANDABLE -->
@@ -236,6 +240,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

bbot/cli.py

@@ -29,7 +29,6 @@
 
 
 async def _main():
-
     import asyncio
     import traceback
     from contextlib import suppress
@@ -45,7 +44,6 @@ async def _main():
     global scan_name
 
     try:
-
         # start by creating a default scan preset
         preset = Preset(_log=True, name="bbot_cli_main")
         # parse command line arguments and merge into preset
@@ -81,7 +79,6 @@ async def _main():
 
         # if we're listing modules or their options
         if options.list_modules or options.list_module_options:
-
             # if no modules or flags are specified, enable everything
             if not (options.modules or options.output_modules or options.flags):
                 for module, preloaded in preset.module_loader.preloaded().items():
@@ -172,7 +169,6 @@ async def _main():
             log.trace(f"Command: {' '.join(sys.argv)}")
 
             if sys.stdin.isatty():
-
                 # warn if any targets belong directly to a cloud provider
                 for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":

bbot/core/config/files.py

@@ -10,7 +10,6 @@
 
 
 class BBOTConfigFiles:
-
     config_dir = (Path.home() / ".config" / "bbot").resolve()
     defaults_filename = (bbot_code_dir / "defaults.yml").resolve()
     config_filename = (config_dir / "bbot.yml").resolve()

bbot/core/event/base.py

@@ -1180,7 +1180,6 @@ def __init__(self, *args, **kwargs):
         self.num_redirects = getattr(self.parent, "num_redirects", 0)
 
     def _data_id(self):
-
         data = super()._data_id()
 
         # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)
@@ -1267,7 +1266,6 @@ def http_status(self):
 
 
 class URL(URL_UNVERIFIED):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -1309,7 +1307,6 @@ class URL_HINT(URL_UNVERIFIED):
 
 
 class WEB_PARAMETER(DictHostEvent):
-
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")

bbot/core/helpers/depsinstaller/installer.py

@@ -49,7 +49,8 @@ def __init__(self, parent_helper):
         self.minimal_git_config.touch()
         os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
 
-        self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
+        self.deps_config = self.parent_helper.config.get("deps", {})
+        self.deps_behavior = self.deps_config.get("behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
         if sys.prefix != sys.base_prefix:

bbot/core/helpers/dns/brute.py

@@ -41,10 +41,13 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
             type = "A"
         type = str(type).strip().upper()
 
-        wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
-        if wildcard_rdtypes:
+        wildcard_domains = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
+        wildcard_rdtypes = set()
+        for domain, rdtypes in wildcard_domains.items():
+            wildcard_rdtypes.update(rdtypes)
+        if wildcard_domains:
             self.log.hugewarning(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(sorted(wildcard_rdtypes))})"
             )
             return []
 

bbot/core/helpers/dns/dns.py

@@ -16,7 +16,6 @@
 
 
 class DNSHelper(EngineClient):
-
     SERVER_CLASS = DNSEngine
     ERROR_CLASS = DNSError
 

bbot/core/helpers/dns/engine.py

@@ -24,7 +24,6 @@
 
 
 class DNSEngine(EngineServer):
-
     CMDS = {
         0: "resolve",
         1: "resolve_raw",
@@ -476,7 +475,6 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         # for every parent domain, starting with the shortest
         parents = list(domain_parents(query))
         for parent in parents[::-1]:
-
             # check if the parent domain is set up with wildcards
             wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)
 

bbot/core/helpers/dns/mock.py

@@ -5,7 +5,6 @@
 
 
 class MockResolver:
-
     def __init__(self, mock_data=None, custom_lookup_fn=None):
         self.mock_data = mock_data if mock_data else {}
         self._custom_lookup_fn = custom_lookup_fn

bbot/core/helpers/libmagic.py

@@ -2,7 +2,6 @@
 
 
 def get_magic_info(file):
-
     magic_detections = puremagic.magic_file(file)
     if magic_detections:
         magic_detections.sort(key=lambda x: x.confidence, reverse=True)
@@ -15,54 +14,52 @@ def get_compression(mime_type):
     mime_type = mime_type.lower()
     # from https://github.com/cdgriffith/puremagic/blob/master/puremagic/magic_data.json
     compression_map = {
-        "application/gzip": "gzip",  # Gzip compressed file
-        "application/zip": "zip",  # Zip archive
-        "application/x-bzip2": "bzip2",  # Bzip2 compressed file
-        "application/x-xz": "xz",  # XZ compressed file
-        "application/x-7z-compressed": "7z",  # 7-Zip archive
-        "application/vnd.rar": "rar",  # RAR archive
-        "application/x-lzma": "lzma",  # LZMA compressed file
-        "application/x-compress": "compress",  # Unix compress file
-        "application/zstd": "zstd",  # Zstandard compressed file
-        "application/x-lz4": "lz4",  # LZ4 compressed file
-        "application/x-tar": "tar",  # Tar archive
-        "application/x-zip-compressed-fb2": "zip",  # Zip archive (FB2)
-        "application/epub+zip": "zip",  # EPUB book (Zip archive)
-        "application/pak": "pak",  # PAK archive
-        "application/x-lha": "lha",  # LHA archive
         "application/arj": "arj",  # ARJ archive
-        "application/vnd.ms-cab-compressed": "cab",  # Microsoft Cabinet archive
-        "application/x-sit": "sit",  # StuffIt archive
         "application/binhex": "binhex",  # BinHex encoded file
-        "application/x-lrzip": "lrzip",  # Long Range ZIP
-        "application/x-alz": "alz",  # ALZip archive
-        "application/x-tgz": "tgz",  # Gzip compressed Tar archive
-        "application/x-gzip": "gzip",  # Gzip compressed file
-        "application/x-lzip": "lzip",  # Lzip compressed file
-        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
-        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
-        "application/vnd.comicbook+zip": "zip",  # Comic book archive (Zip)
-        "application/vnd.palm": "palm",  # Palm OS data
+        "application/epub+zip": "zip",  # EPUB book (Zip archive)
         "application/fictionbook2+zip": "zip",  # FictionBook 2.0 (Zip)
         "application/fictionbook3+zip": "zip",  # FictionBook 3.0 (Zip)
+        "application/gzip": "gzip",  # Gzip compressed file
+        "application/java-archive": "zip",  # Java Archive (JAR)
+        "application/pak": "pak",  # PAK archive
+        "application/vnd.android.package-archive": "zip",  # Android package (APK)
+        "application/vnd.comicbook-rar": "rar",  # Comic book archive (RAR)
+        "application/vnd.comicbook+zip": "zip",  # Comic book archive (Zip)
+        "application/vnd.ms-cab-compressed": "cab",  # Microsoft Cabinet archive
+        "application/vnd.palm": "palm",  # Palm OS data
+        "application/vnd.rar": "rar",  # RAR archive
+        "application/x-7z-compressed": "7z",  # 7-Zip archive
+        "application/x-ace": "ace",  # ACE archive
+        "application/x-alz": "alz",  # ALZip archive
+        "application/x-arc": "arc",  # ARC archive
+        "application/x-archive": "ar",  # Unix archive
+        "application/x-bzip2": "bzip2",  # Bzip2 compressed file
+        "application/x-compress": "compress",  # Unix compress file
         "application/x-cpio": "cpio",  # CPIO archive
+        "application/x-gzip": "gzip",  # Gzip compressed file
+        "application/x-itunes-ipa": "zip",  # iOS application archive (IPA)
         "application/x-java-pack200": "pack200",  # Java Pack200 archive
+        "application/x-lha": "lha",  # LHA archive
+        "application/x-lrzip": "lrzip",  # Long Range ZIP
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/x-lz4": "lz4",  # LZ4 compressed file
+        "application/x-lzip": "lzip",  # Lzip compressed file
+        "application/x-lzma": "lzma",  # LZMA compressed file
         "application/x-par2": "par2",  # PAR2 recovery file
+        "application/x-qpress": "qpress",  # Qpress archive
         "application/x-rar-compressed": "rar",  # RAR archive
-        "application/java-archive": "zip",  # Java Archive (JAR)
-        "application/x-webarchive": "zip",  # Web archive (Zip)
-        "application/vnd.android.package-archive": "zip",  # Android package (APK)
-        "application/x-itunes-ipa": "zip",  # iOS application archive (IPA)
+        "application/x-sit": "sit",  # StuffIt archive
         "application/x-stuffit": "sit",  # StuffIt archive
-        "application/x-archive": "ar",  # Unix archive
-        "application/x-qpress": "qpress",  # Qpress archive
+        "application/x-tar": "tar",  # Tar archive
+        "application/x-tgz": "tgz",  # Gzip compressed Tar archive
+        "application/x-webarchive": "zip",  # Web archive (Zip)
         "application/x-xar": "xar",  # XAR archive
-        "application/x-ace": "ace",  # ACE archive
+        "application/x-xz": "xz",  # XZ compressed file
+        "application/x-zip-compressed-fb2": "zip",  # Zip archive (FB2)
         "application/x-zoo": "zoo",  # Zoo archive
-        "application/x-arc": "arc",  # ARC archive
         "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
-        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
-        "application/vnd.comicbook-rar": "rar",  # Comic book archive (RAR)
+        "application/zip": "zip",  # Zip archive
+        "application/zstd": "zstd",  # Zstandard compressed file
     }
 
     return compression_map.get(mime_type, "")

bbot/core/helpers/process.py

@@ -7,7 +7,6 @@
 
 
 class BBOTThread(threading.Thread):
-
     default_name = "default bbot thread"
 
     def __init__(self, *args, **kwargs):
@@ -24,7 +23,6 @@ def run(self):
 
 
 class BBOTProcess(SpawnProcess):
-
     default_name = "bbot process pool"
 
     def __init__(self, *args, **kwargs):

bbot/core/helpers/validators.py

@@ -299,7 +299,6 @@ def is_email(email):
 
 
 class Validators:
-
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
 

bbot/core/helpers/web/engine.py

@@ -14,7 +14,6 @@
 
 
 class HTTPEngine(EngineServer):
-
     CMDS = {
         0: "request",
         1: "request_batch",

bbot/core/helpers/web/web.py

@@ -19,7 +19,6 @@
 
 
 class WebHelper(EngineClient):
-
     SERVER_CLASS = HTTPEngine
     ERROR_CLASS = WebError
 

bbot/core/modules.py

@@ -337,7 +337,6 @@ def preload_module(self, module_file):
             # look for classes
             if type(root_element) == ast.ClassDef:
                 for class_attr in root_element.body:
-
                     # class attributes that are dictionaries
                     if type(class_attr) == ast.Assign and type(class_attr.value) == ast.Dict:
                         # module options