Merge pull request #1650 from blacklanternsecurity/dev

Dev --> Stable (2.01)

.github/workflows/version_updater.yaml

@@ -28,8 +28,9 @@ jobs:
           response = requests.get('https://api.github.com/repos/projectdiscovery/nuclei/releases/latest')
           version = response.json()['tag_name'].lstrip('v')
           release_notes = response.json()['body']
-          os.system(f"echo 'latest_version={version}' >> $GITHUB_ENV")
-          os.system(f"echo 'release_notes={release_notes}' >> $GITHUB_ENV")
+          with open(os.getenv('GITHUB_ENV'), 'a') as env_file:
+            env_file.write(f"latest_version={version}\n")
+            env_file.write(f"release_notes<<EOF\n{release_notes}\nEOF\n")
         shell: python
       - name: Get current version
         id: get-current-version
@@ -44,13 +45,13 @@ jobs:
         if: steps.update-version.outcome == 'success'
         uses: peter-evans/create-pull-request@v5
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update nuclei"
           title: "Update nuclei to ${{ env.latest_version }}"
           body: |
             This PR uses https://api.github.com/repos/projectdiscovery/nuclei/releases/latest to obtain the latest version of nuclei and update the version in bbot/modules/deadly/nuclei.py."
 
-            Release notes:
+            # Release notes:
             ${{ env.release_notes }}
           branch: "update-nuclei"
           committer: GitHub <[email protected]>
@@ -78,8 +79,9 @@ jobs:
           response = requests.get('https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest')
           version = response.json()['tag_name'].lstrip('v')
           release_notes = response.json()['body']
-          os.system(f"echo 'latest_version={version}' >> $GITHUB_ENV")
-          os.system(f"echo 'release_notes={release_notes}' >> $GITHUB_ENV")
+          with open(os.getenv('GITHUB_ENV'), 'a') as env_file:
+            env_file.write(f"latest_version={version}\n")
+            env_file.write(f"release_notes<<EOF\n{release_notes}\nEOF\n")
         shell: python
       - name: Get current version
         id: get-current-version
@@ -94,13 +96,13 @@ jobs:
         if: steps.update-version.outcome == 'success'
         uses: peter-evans/create-pull-request@v5
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update trufflehog"
           title: "Update trufflehog to ${{ env.latest_version }}"
           body: |
             This PR uses https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest to obtain the latest version of trufflehog and update the version in bbot/modules/trufflehog.py.
 
-            Release notes:
+            # Release notes:
             ${{ env.release_notes }}
           branch: "update-trufflehog"
           committer: GitHub <[email protected]>

README.md

@@ -29,6 +29,9 @@ Passive API sources plus a recursive DNS brute-force with target-specific subdom
 ```bash
 # find subdomains of evilcorp.com
 bbot -t evilcorp.com -p subdomain-enum
+
+# passive sources only
+bbot -t evilcorp.com -p subdomain-enum -rf passive
 ```
 
 <!-- BBOT SUBDOMAIN-ENUM PRESET EXPANDABLE -->

bbot/core/event/base.py

@@ -127,7 +127,7 @@ def __init__(
         scan=None,
         scans=None,
         tags=None,
-        confidence=5,
+        confidence=100,
         timestamp=None,
         _dummy=False,
         _internal=None,
@@ -146,7 +146,7 @@ def __init__(
             scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None.
             scans (list of Scan, optional): BBOT Scan objects, used primarily when unserializing an Event from the database. Defaults to None.
             tags (list of str, optional): Descriptive tags for the event. Defaults to None.
-            confidence (int, optional): Confidence level for the event, on a scale of 1-10. Defaults to 5.
+            confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.
             timestamp (datetime, optional): Time of event discovery. Defaults to current UTC time.
             _dummy (bool, optional): If True, disables certain data validations. Defaults to False.
             _internal (Any, optional): If specified, makes the event internal. Defaults to None.
@@ -237,6 +237,27 @@ def __init__(
     def data(self):
         return self._data
 
+    @property
+    def confidence(self):
+        return self._confidence
+
+    @confidence.setter
+    def confidence(self, confidence):
+        self._confidence = min(100, max(1, int(confidence)))
+
+    @property
+    def cumulative_confidence(self):
+        """
+        Considers the confidence of parent events. This is useful for filtering out speculative/unreliable events.
+
+        E.g. an event with a confidence of 50 whose parent is also 50 would have a cumulative confidence of 25.
+
+        A confidence of 100 will reset the cumulative confidence to 100.
+        """
+        if self._confidence == 100 or self.parent is None or self.parent is self:
+            return self._confidence
+        return int(self._confidence * self.parent.cumulative_confidence / 100)
+
     @property
     def resolved_hosts(self):
         if is_ip(self.host):
@@ -359,7 +380,7 @@ def discovery_path(self):
         This event's full discovery context, including those of all its parents
         """
         parent_path = []
-        if self.parent is not None and self != self.parent:
+        if self.parent is not None and self.parent is not self:
             parent_path = self.parent.discovery_path
         return parent_path + [[self.id, self.discovery_context]]
 
@@ -387,6 +408,10 @@ def tags(self, tags):
     def add_tag(self, tag):
         self._tags.add(tagify(tag))
 
+    def add_tags(self, tags):
+        for tag in set(tags):
+            self.add_tag(tag)
+
     def remove_tag(self, tag):
         with suppress(KeyError):
             self._tags.remove(tagify(tag))
@@ -461,10 +486,10 @@ def scope_distance(self, scope_distance):
                 self.remove_tag("in-scope")
                 self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
-        # apply recursively to parent events
-        parent_scope_distance = getattr(self.parent, "scope_distance", None)
-        if parent_scope_distance is not None and self != self.parent:
-            self.parent.scope_distance = scope_distance + 1
+            # apply recursively to parent events
+            parent_scope_distance = getattr(self.parent, "scope_distance", None)
+            if parent_scope_distance is not None and self.parent is not self:
+                self.parent.scope_distance = new_scope_distance + 1
 
     @property
     def scope_description(self):
@@ -869,7 +894,7 @@ def __hash__(self):
 
     def __str__(self):
         max_event_len = 80
-        d = str(self.data)
+        d = str(self.data).replace("\n", "\\n")
         return f'{self.type}("{d[:max_event_len]}{("..." if len(d) > max_event_len else "")}", module={self.module}, tags={self.tags})'
 
     def __repr__(self):
@@ -923,19 +948,40 @@ def _host(self):
                 return make_ip_type(parsed.hostname)
 
 
-class DictPathEvent(DictEvent):
-    _path_keywords = ["path", "filename"]
+class ClosestHostEvent(DictHostEvent):
+    # if a host/path/url isn't specified, this event type grabs it from the closest parent
+    # inherited by FINDING and VULNERABILITY
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not self.host:
+            for parent in self.get_parents(include_self=True):
+                # inherit closest URL
+                if not "url" in self.data:
+                    parent_url = getattr(parent, "parsed_url", None)
+                    if parent_url is not None:
+                        self.data["url"] = parent_url.geturl()
+                # inherit closest path
+                if not "path" in self.data and isinstance(parent.data, dict):
+                    parent_path = parent.data.get("path", None)
+                    if parent_path is not None:
+                        self.data["path"] = parent_path
+                # inherit closest host
+                if parent.host:
+                    self.data["host"] = str(parent.host)
+                    break
+        # die if we still haven't found a host
+        if not self.host:
+            raise ValueError("No host was found in event parents. Host must be specified!")
 
+
+class DictPathEvent(DictEvent):
     def sanitize_data(self, data):
         new_data = dict(data)
         file_blobs = getattr(self.scan, "_file_blobs", False)
         folder_blobs = getattr(self.scan, "_folder_blobs", False)
-        for path_keyword in self._path_keywords:
-            blob = None
-            try:
-                data_path = Path(data[path_keyword])
-            except KeyError:
-                continue
+        blob = None
+        try:
+            data_path = Path(data["path"])
             if data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
@@ -945,10 +991,10 @@ def sanitize_data(self, data):
                 self.add_tag("folder")
                 if folder_blobs:
                     blob = self._tar_directory(data_path)
-            else:
-                continue
-            if blob:
-                new_data["blob"] = base64.b64encode(blob).decode("utf-8")
+        except KeyError:
+            pass
+        if blob:
+            new_data["blob"] = base64.b64encode(blob).decode("utf-8")
 
         return new_data
 
@@ -1300,7 +1346,7 @@ def redirect_location(self):
         return location
 
 
-class VULNERABILITY(DictHostEvent):
+class VULNERABILITY(ClosestHostEvent):
     _always_emit = True
     _quick_emit = True
     severity_colors = {
@@ -1316,10 +1362,11 @@ def sanitize_data(self, data):
         return data
 
     class _data_validator(BaseModel):
-        host: str
+        host: Optional[str] = None
         severity: str
         description: str
         url: Optional[str] = None
+        path: Optional[str] = None
         _validate_url = field_validator("url")(validators.validate_url)
         _validate_host = field_validator("host")(validators.validate_host)
         _validate_severity = field_validator("severity")(validators.validate_severity)
@@ -1328,14 +1375,15 @@ def _pretty_string(self):
         return f'[{self.data["severity"]}] {self.data["description"]}'
 
 
-class FINDING(DictHostEvent):
+class FINDING(ClosestHostEvent):
     _always_emit = True
     _quick_emit = True
 
     class _data_validator(BaseModel):
-        host: str
+        host: Optional[str] = None
         description: str
         url: Optional[str] = None
+        path: Optional[str] = None
         _validate_url = field_validator("url")(validators.validate_url)
         _validate_host = field_validator("host")(validators.validate_host)
 
@@ -1464,7 +1512,7 @@ def make_event(
     scan=None,
     scans=None,
     tags=None,
-    confidence=5,
+    confidence=100,
     dummy=False,
     internal=None,
 ):
@@ -1484,7 +1532,7 @@ def make_event(
         scan (Scan, optional): BBOT Scan object associated with the event.
         scans (List[Scan], optional): Multiple BBOT Scan objects, primarily used for unserialization.
         tags (Union[str, List[str]], optional): Descriptive tags for the event, as a list or a single string.
-        confidence (int, optional): Confidence level for the event, on a scale of 1-10. Defaults to 5.
+        confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.
         dummy (bool, optional): Disables data validations if set to True. Defaults to False.
         internal (Any, optional): Makes the event internal if set to True. Defaults to None.
 
@@ -1613,7 +1661,7 @@ def event_from_json(j, siem_friendly=False):
             "event_type": event_type,
             "scans": j.get("scans", []),
             "tags": j.get("tags", []),
-            "confidence": j.get("confidence", 5),
+            "confidence": j.get("confidence", 100),
             "context": j.get("discovery_context", None),
             "dummy": True,
         }

bbot/core/helpers/command.py

@@ -3,9 +3,9 @@
 import logging
 import traceback
 from signal import SIGINT
-from subprocess import CompletedProcess, CalledProcessError
+from subprocess import CompletedProcess, CalledProcessError, SubprocessError
 
-from .misc import smart_decode, smart_encode
+from .misc import smart_decode, smart_encode, which
 
 log = logging.getLogger("bbot.core.helpers.command")
 
@@ -182,7 +182,11 @@ async def _spawn_proc(self, *command, **kwargs):
         >>> _spawn_proc("ls", "-l", input="data")
         (<Process ...>, "data", ["ls", "-l"])
     """
-    command, kwargs = self._prepare_command_kwargs(command, kwargs)
+    try:
+        command, kwargs = self._prepare_command_kwargs(command, kwargs)
+    except SubprocessError as e:
+        log.warning(e)
+        return None, None, None
     _input = kwargs.pop("input", None)
     if _input is not None:
         if kwargs.get("stdin") is not None:
@@ -276,6 +280,17 @@ def _prepare_command_kwargs(self, command, kwargs):
         command = command[0]
     command = [str(s) for s in command]
 
+    if not command:
+        raise SubprocessError("Must specify a command")
+
+    # use full path of binary, if not already specified
+    binary = command[0]
+    if not "/" in binary:
+        binary_full_path = which(binary)
+        if binary_full_path is None:
+            raise SubprocessError(f'Command "{binary}" was not found')
+        command[0] = binary_full_path
+
     env = kwargs.get("env", os.environ)
     if sudo and os.geteuid() != 0:
         self.depsinstaller.ensure_root()

bbot/core/helpers/dns/dns.py

@@ -56,7 +56,8 @@ def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.config = self.parent_helper.config
         self.dns_config = self.config.get("dns", {})
-        super().__init__(server_kwargs={"config": self.config})
+        engine_debug = self.config.get("engine", {}).get("debug", False)
+        super().__init__(server_kwargs={"config": self.config}, debug=engine_debug)
 
         # resolver
         self.timeout = self.dns_config.get("timeout", 5)