Merge branch 'bbot-2.0' into new-presets

.github/workflows/version_updater.yaml

@@ -0,0 +1,108 @@
+name: Version Updater
+on:
+  schedule:
+    # Runs at 00:00 every day
+    - cron: '0 0 * * *'
+  workflow_dispatch: # Adds the ability to manually trigger the workflow
+
+jobs:
+  update-nuclei-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: dev
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+      - name: Get latest version
+        id: get-latest-version
+        run: |
+          import os, requests
+          response = requests.get('https://api.github.com/repos/projectdiscovery/nuclei/releases/latest')
+          version = response.json()['tag_name'].lstrip('v')
+          release_notes = response.json()['body']
+          os.system(f"echo 'latest_version={version}' >> $GITHUB_ENV")
+          os.system(f"echo 'release_notes={release_notes}' >> $GITHUB_ENV")
+        shell: python
+      - name: Get current version
+        id: get-current-version
+        run: |
+          version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/deadly/nuclei.py)
+          echo "current_version=$version" >> $GITHUB_ENV
+      - name: Update version
+        id: update-version
+        if: env.latest_version != env.current_version
+        run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/deadly/nuclei.py"
+      - name: Create pull request to update the version
+        if: steps.update-version.outcome == 'success'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Update nuclei"
+          title: "Update nuclei to ${{ env.latest_version }}"
+          body: |
+            This PR uses https://api.github.com/repos/projectdiscovery/nuclei/releases/latest to obtain the latest version of nuclei and update the version in bbot/modules/deadly/nuclei.py."
+
+            Release notes:
+            ${{ env.release_notes }}
+          branch: "update-nuclei"
+          committer: GitHub <[email protected]>
+          author: GitHub <[email protected]>
+          assignees: "TheTechromancer"
+  update-trufflehog-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: dev
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+      - name: Get latest version
+        id: get-latest-version
+        run: |
+          import os, requests
+          response = requests.get('https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest')
+          version = response.json()['tag_name'].lstrip('v')
+          release_notes = response.json()['body']
+          os.system(f"echo 'latest_version={version}' >> $GITHUB_ENV")
+          os.system(f"echo 'release_notes={release_notes}' >> $GITHUB_ENV")
+        shell: python
+      - name: Get current version
+        id: get-current-version
+        run: |
+          version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/trufflehog.py)
+          echo "current_version=$version" >> $GITHUB_ENV
+      - name: Update version
+        id: update-version
+        if: env.latest_version != env.current_version
+        run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/trufflehog.py"
+      - name: Create pull request to update the version
+        if: steps.update-version.outcome == 'success'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Update trufflehog"
+          title: "Update trufflehog to ${{ env.latest_version }}"
+          body: |
+            This PR uses https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest to obtain the latest version of trufflehog and update the version in bbot/modules/trufflehog.py.
+
+            Release notes:
+            ${{ env.release_notes }}
+          branch: "update-trufflehog"
+          committer: GitHub <[email protected]>
+          author: GitHub <[email protected]>
+          assignees: "TheTechromancer"

bbot/cli.py

@@ -168,6 +168,15 @@ async def _main():
             log.trace(f"Command: {' '.join(sys.argv)}")
 
             if sys.stdin.isatty():
+
+                # warn if any targets belong directly to a cloud provider
+                for event in scan.target.events:
+                    if event.type == "DNS_NAME":
+                        if scan.helpers.cloudcheck(event.host):
+                            scan.hugewarning(
+                                f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!'
+                            )
+
                 if not options.yes:
                     log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}")
                     input()

bbot/core/event/base.py

@@ -1,15 +1,20 @@
+import io
 import re
 import json
+import base64
 import logging
+import tarfile
 import datetime
 import ipaddress
 import traceback
+
 from copy import copy
 from typing import Optional
 from contextlib import suppress
 from urllib.parse import urljoin
 from radixtarget import RadixTarget
 from pydantic import BaseModel, field_validator
+from pathlib import Path
 
 from .helpers import *
 from bbot.errors import *
@@ -842,6 +847,43 @@ def _host(self):
                 return make_ip_type(parsed.hostname)
 
 
+class DictPathEvent(DictEvent):
+    _path_keywords = ["path", "filename"]
+
+    def sanitize_data(self, data):
+        new_data = dict(data)
+        file_blobs = getattr(self.scan, "_file_blobs", False)
+        folder_blobs = getattr(self.scan, "_folder_blobs", False)
+        for path_keyword in self._path_keywords:
+            blob = None
+            try:
+                data_path = Path(data[path_keyword])
+            except KeyError:
+                continue
+            if data_path.is_file():
+                self.add_tag("file")
+                if file_blobs:
+                    with open(data_path, "rb") as file:
+                        blob = file.read()
+            elif data_path.is_dir():
+                self.add_tag("folder")
+                if folder_blobs:
+                    blob = self._tar_directory(data_path)
+            else:
+                continue
+            if blob:
+                new_data["blob"] = base64.b64encode(blob).decode("utf-8")
+
+        return new_data
+
+    def _tar_directory(self, dir_path):
+        tar_buffer = io.BytesIO()
+        with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar:
+            # Add the entire directory to the tar archive
+            tar.add(dir_path, arcname=dir_path.name)
+        return tar_buffer.getvalue()
+
+
 class ASN(DictEvent):
     _always_emit = True
     _quick_emit = True
@@ -1241,7 +1283,7 @@ class SOCIAL(DictHostEvent):
     _scope_distance_increment_same_host = True
 
 
-class WEBSCREENSHOT(DictHostEvent):
+class WEBSCREENSHOT(DictPathEvent, DictHostEvent):
     _always_emit = True
     _quick_emit = True
 
@@ -1267,6 +1309,10 @@ def _pretty_string(self):
         return self.data["waf"]
 
 
+class FILESYSTEM(DictPathEvent):
+    pass
+
+
 def make_event(
     data,
     event_type=None,

bbot/core/helpers/dns/dns.py

@@ -81,6 +81,9 @@ def __init__(self, parent_helper):
     async def resolve(self, query, **kwargs):
         return await self.run_and_return("resolve", query=query, **kwargs)
 
+    async def resolve_raw(self, query, **kwargs):
+        return await self.run_and_return("resolve_raw", query=query, **kwargs)
+
     async def resolve_batch(self, queries, **kwargs):
         async for _ in self.run_and_yield("resolve_batch", queries=queries, **kwargs):
             yield _

bbot/core/helpers/dns/engine.py

@@ -30,10 +30,11 @@ class DNSEngine(EngineServer):
 
     CMDS = {
         0: "resolve",
-        1: "resolve_batch",
-        2: "resolve_raw_batch",
-        3: "is_wildcard",
-        4: "is_wildcard_domain",
+        1: "resolve_raw",
+        2: "resolve_batch",
+        3: "resolve_raw_batch",
+        4: "is_wildcard",
+        5: "is_wildcard_domain",
         99: "_mock_dns",
     }
 
@@ -113,7 +114,6 @@ async def resolve(self, query, **kwargs):
             {"1.2.3.4", "dead::beef"}
         """
         results = set()
-        errors = []
         try:
             answers, errors = await self.resolve_raw(query, **kwargs)
             for answer in answers:

bbot/core/helpers/misc.py

@@ -2484,7 +2484,7 @@ async def cancel_tasks(tasks, ignore_errors=True):
     current_task = asyncio.current_task()
     tasks = [t for t in tasks if t != current_task]
     for task in tasks:
-        log.debug(f"Cancelling task: {task}")
+        # log.debug(f"Cancelling task: {task}")
         task.cancel()
     if ignore_errors:
         for task in tasks:
@@ -2516,7 +2516,7 @@ def cancel_tasks_sync(tasks):
     current_task = asyncio.current_task()
     for task in tasks:
         if task != current_task:
-            log.debug(f"Cancelling task: {task}")
+            # log.debug(f"Cancelling task: {task}")
             task.cancel()
 
 

bbot/defaults.yml

@@ -18,6 +18,10 @@ status_frequency: 15
 http_proxy: 
 # Web user-agent
 user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97
+# Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event
+file_blobs: false
+# Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event
+folder_blobs: false
 
 # Tool dependencies
 deps:
@@ -150,8 +154,10 @@ url_extension_httpx_only:
 # Don't output these types of events (they are still distributed to modules)
 omit_event_types:
     - HTTP_RESPONSE
+    - RAW_TEXT
     - URL_UNVERIFIED
     - DNS_NAME_UNRESOLVED
+    - FILESYSTEM
     # - IP_ADDRESS
 
 # Custom interactsh server settings
@@ -165,3 +171,7 @@ dns_omit_queries:
   - SRV:mail.protection.outlook.com
   - CNAME:mail.protection.outlook.com
   - TXT:mail.protection.outlook.com
+
+# temporary fix to boost scan performance
+# TODO: remove this when https://github.com/blacklanternsecurity/bbot/issues/1252 is merged
+target_dns_regex_disable: false

bbot/modules/bucket_file_enum.py

@@ -4,13 +4,15 @@
 
 class bucket_file_enum(BaseModule):
     """
-    Enumerate files in a public bucket
+    Enumerate files in public storage buckets
+
+    Currently only supports AWS and DigitalOcean
     """
 
     watched_events = ["STORAGE_BUCKET"]
     produced_events = ["URL_UNVERIFIED"]
     meta = {
-        "description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS",
+        "description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS, DigitalOcean",
         "created_date": "2023-11-14",
         "author": "@TheTechromancer",
     }