Merge branch 'dev' into extract_internal_module

.gitattributes

@@ -5,4 +5,4 @@
 *.txt text eol=lf
 *.json text eol=lf
 *.md text eol=lf
-*.sh text eol=lf
+*.sh text eol=lf

.github/dependabot.yml

@@ -14,3 +14,4 @@ updates:
           - "*"  # Group all Actions updates into a single larger pull request
     schedule:
       interval: weekly
+    target-branch: "dev" 

.github/workflows/tests.yml

@@ -15,19 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: psf/black@stable
-        with:
-          options: "--check"
-      - name: Install Python 3
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          pip install flake8
-      - name: flake8
-        run: |
-          flake8
+      - run: |
+          pipx install ruff
+          ruff check
+          ruff format
   test:
     needs: lint
     runs-on: ubuntu-latest

.github/workflows/version_updater.yml

@@ -9,13 +9,13 @@ jobs:
   update-nuclei-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -44,7 +44,7 @@ jobs:
         run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/deadly/nuclei.py"
       - name: Create pull request to update the version
         if: steps.update-version.outcome == 'success'
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update nuclei"
@@ -61,13 +61,13 @@ jobs:
   update-trufflehog-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -96,7 +96,7 @@ jobs:
         run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/trufflehog.py"
       - name: Create pull request to update the version
         if: steps.update-version.outcome == 'success'
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update trufflehog"
@@ -109,4 +109,4 @@ jobs:
           branch: "update-trufflehog"
           committer: blsaccess <[email protected]>
           author: blsaccess <[email protected]>
-          assignees: "TheTechromancer"
+          assignees: "TheTechromancer"

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "bbot/modules/playground"]
     path = bbot/modules/playground
     url = https://github.com/blacklanternsecurity/bbot-module-playground
-    branch = main
+    branch = main

.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+# Learn more about this config here: https://pre-commit.com/
+
+# To enable these pre-commit hooks run:
+# `pipx install pre-commit` or `brew install pre-commit`
+# Then in the project root directory run `pre-commit install`
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-builtin-literals
+      - id: check-byte-order-marker
+      - id: check-case-conflict
+      # - id: check-docstring-first
+      # - id: check-executables-have-shebangs
+      - id: check-json
+      - id: check-merge-conflict
+      # - id: check-shebang-scripts-are-executable
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-vcs-permalinks
+      - id: check-xml
+      # - id: check-yaml
+      - id: debug-statements
+      - id: destroyed-symlinks
+      # - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: file-contents-sorter
+      - id: fix-byte-order-marker
+      - id: forbid-new-submodules
+      - id: forbid-submodules
+      - id: mixed-line-ending
+      - id: requirements-txt-fixer
+      - id: sort-simple-yaml
+      - id: trailing-whitespace
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.0
+    hooks:
+      - id: ruff
+      - id: ruff-format
+
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.23
+    hooks:
+      - id: validate-pyproject

README.md

@@ -1,6 +1,6 @@
 [![bbot_banner](https://github.com/user-attachments/assets/f02804ce-9478-4f1e-ac4d-9cf5620a3214)](https://github.com/blacklanternsecurity/bbot)
 
-[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
+[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
 
 ### **BEE·bot** is a multipurpose scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), built to automate your **Recon**, **Bug Bounties**, and **ASM**!
 

bbot-docker.sh

@@ -1,2 +1,3 @@
-# run the docker image
-docker run --rm -it -v "$HOME/.bbot:/root/.bbot" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@"
+# OUTPUTS SCAN DATA TO ~/.bbot/scans
+
+docker run --rm -it -v "$HOME/.bbot/scans:/root/.bbot/scans" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@"

bbot/cli.py

@@ -29,7 +29,6 @@
 
 
 async def _main():
-
     import asyncio
     import traceback
     from contextlib import suppress
@@ -45,7 +44,6 @@ async def _main():
     global scan_name
 
     try:
-
         # start by creating a default scan preset
         preset = Preset(_log=True, name="bbot_cli_main")
         # parse command line arguments and merge into preset
@@ -80,8 +78,7 @@ async def _main():
             return
 
         # if we're listing modules or their options
-        if options.list_modules or options.list_module_options:
-
+        if options.list_modules or options.list_output_modules or options.list_module_options:
             # if no modules or flags are specified, enable everything
             if not (options.modules or options.output_modules or options.flags):
                 for module, preloaded in preset.module_loader.preloaded().items():
@@ -99,7 +96,17 @@ async def _main():
                 print("")
                 print("### MODULES ###")
                 print("")
-                for row in preset.module_loader.modules_table(preset.modules).splitlines():
+                modules = sorted(set(preset.scan_modules + preset.internal_modules))
+                for row in preset.module_loader.modules_table(modules).splitlines():
+                    print(row)
+                return
+
+            # --list-output-modules
+            if options.list_output_modules:
+                print("")
+                print("### OUTPUT MODULES ###")
+                print("")
+                for row in preset.module_loader.modules_table(preset.output_modules).splitlines():
                     print(row)
                 return
 
@@ -133,8 +140,8 @@ async def _main():
         ]
         if deadly_modules and not options.allow_deadly:
             log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}")
-            log.hugewarning(f"Deadly modules are highly intrusive")
-            log.hugewarning(f"Please specify --allow-deadly to continue")
+            log.hugewarning("Deadly modules are highly intrusive")
+            log.hugewarning("Please specify --allow-deadly to continue")
             return False
 
         # --current-preset
@@ -172,7 +179,6 @@ async def _main():
             log.trace(f"Command: {' '.join(sys.argv)}")
 
             if sys.stdin.isatty():
-
                 # warn if any targets belong directly to a cloud provider
                 for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":

bbot/core/config/files.py

@@ -10,7 +10,6 @@
 
 
 class BBOTConfigFiles:
-
     config_dir = (Path.home() / ".config" / "bbot").resolve()
     defaults_filename = (bbot_code_dir / "defaults.yml").resolve()
     config_filename = (config_dir / "bbot.yml").resolve()

bbot/core/config/logger.py

@@ -68,7 +68,7 @@ def __init__(self, core):
         self.listener = None
 
         # if we haven't set up logging yet, do it now
-        if not "_BBOT_LOGGING_SETUP" in os.environ:
+        if "_BBOT_LOGGING_SETUP" not in os.environ:
             os.environ["_BBOT_LOGGING_SETUP"] = "1"
             self.queue = multiprocessing.Queue()
             self.setup_queue_handler()

bbot/core/core.py

@@ -106,7 +106,7 @@ def default_config(self):
         if DEFAULT_CONFIG is None:
             self.default_config = self.files_config.get_default_config()
             # ensure bbot home dir
-            if not "home" in self.default_config:
+            if "home" not in self.default_config:
                 self.default_config["home"] = "~/.bbot"
         return DEFAULT_CONFIG
 

bbot/core/event/base.py

@@ -175,8 +175,8 @@ def __init__(
         self._scope_distance = None
         self._module_priority = None
         self._resolved_hosts = set()
-        self.dns_children = dict()
-        self.raw_dns_records = dict()
+        self.dns_children = {}
+        self.raw_dns_records = {}
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
@@ -203,7 +203,7 @@ def __init__(
         # self.scan holds the instantiated scan object (for helpers, etc.)
         self.scan = scan
         if (not self.scan) and (not self._dummy):
-            raise ValidationError(f"Must specify scan")
+            raise ValidationError("Must specify scan")
         # self.scans holds a list of scan IDs from scans that encountered this event
         self.scans = []
         if scans is not None:
@@ -222,7 +222,7 @@ def __init__(
 
         self.parent = parent
         if (not self.parent) and (not self._dummy):
-            raise ValidationError(f"Must specify event parent")
+            raise ValidationError("Must specify event parent")
 
         if tags is not None:
             for tag in tags:
@@ -301,9 +301,9 @@ def internal(self, value):
         The purpose of internal events is to enable speculative/explorative discovery without cluttering
         the console with irrelevant or uninteresting events.
         """
-        if not value in (True, False):
+        if value not in (True, False):
             raise ValueError(f'"internal" must be boolean, not {type(value)}')
-        if value == True:
+        if value is True:
             self.add_tag("internal")
         else:
             self.remove_tag("internal")
@@ -769,7 +769,7 @@ def json(self, mode="json", siem_friendly=False):
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
         """
-        j = dict()
+        j = {}
         # type, ID, scope description
         for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
@@ -1013,12 +1013,12 @@ def __init__(self, *args, **kwargs):
         if not self.host:
             for parent in self.get_parents(include_self=True):
                 # inherit closest URL
-                if not "url" in self.data:
+                if "url" not in self.data:
                     parent_url = getattr(parent, "parsed_url", None)
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
+                if "path" not in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
@@ -1180,7 +1180,6 @@ def __init__(self, *args, **kwargs):
         self.num_redirects = getattr(self.parent, "num_redirects", 0)
 
     def _data_id(self):
-
         data = super()._data_id()
 
         # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)
@@ -1228,7 +1227,7 @@ def sanitize_data(self, data):
 
     def add_tag(self, tag):
         host_same_as_parent = self.parent and self.host == self.parent.host
-        if tag == "spider-danger" and host_same_as_parent and not "spider-danger" in self.tags:
+        if tag == "spider-danger" and host_same_as_parent and "spider-danger" not in self.tags:
             # increment the web spider distance
             if self.type == "URL_UNVERIFIED":
                 self.web_spider_distance += 1
@@ -1250,7 +1249,7 @@ def with_port(self):
 
     def _words(self):
         first_elem = self.parsed_url.path.lstrip("/").split("/")[0]
-        if not "." in first_elem:
+        if "." not in first_elem:
             return extract_words(first_elem)
         return set()
 
@@ -1267,7 +1266,6 @@ def http_status(self):
 
 
 class URL(URL_UNVERIFIED):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -1279,7 +1277,7 @@ def __init__(self, *args, **kwargs):
     @property
     def resolved_hosts(self):
         # TODO: remove this when we rip out httpx
-        return set(".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-"))
+        return {".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")}
 
     @property
     def pretty_string(self):
@@ -1309,7 +1307,6 @@ class URL_HINT(URL_UNVERIFIED):
 
 
 class WEB_PARAMETER(DictHostEvent):
-
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")
@@ -1668,7 +1665,7 @@ def make_event(
             event.parent = parent
         if context is not None:
             event.discovery_context = context
-        if internal == True:
+        if internal is True:
             event.internal = True
         if tags:
             event.tags = tags.union(event.tags)