File Download Module

bbot/cli.py

@@ -298,7 +298,7 @@ async def _main():
 
                 scanner.helpers.word_cloud.load()
 
-                await scanner.prep()
+                await scanner._prep()
 
                 if not options.dry_run:
                     if not options.agent_mode and not options.yes and sys.stdin.isatty():

bbot/core/event/helpers.py

@@ -12,7 +12,21 @@
 
 def get_event_type(data):
     """
-    Attempt to divine event type from data
+    Determines the type of event based on the given data.
+
+    Args:
+        data (str): The data to be used for determining the event type.
+
+    Returns:
+        str: The type of event such as "IP_ADDRESS", "IP_RANGE", or "URL_UNVERIFIED".
+
+    Raises:
+        ValidationError: If the event type could not be determined.
+
+    Notes:
+        - Utilizes `smart_decode_punycode` and `smart_decode` to preprocess the data.
+        - Makes use of `ipaddress` standard library to check for IP and network types.
+        - Checks against a set of predefined regular expressions stored in `event_type_regexes`.
     """
     data = smart_decode_punycode(smart_decode(data).strip())
 

bbot/core/helpers/command.py

@@ -10,10 +10,28 @@
 
 
 async def run(self, *command, check=False, text=True, **kwargs):
-    """
-    Simple helper for running a command, and getting its output as a string
-        process = await run(["ls", "/tmp"])
-        process.stdout --> "file1.txt\nfile2.txt"
+    """Runs a command asynchronously and gets its output as a string.
+
+    This method is a simple helper for executing a command and capturing its output.
+    If an error occurs during execution, it can optionally raise an error or just log the stderr.
+
+    Args:
+        *command (str): The command to run as separate arguments.
+        check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.
+                                Defaults to False.
+        text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.
+        **kwargs (dict): Additional keyword arguments for the subprocess.
+
+    Returns:
+        CompletedProcess: A completed process object with attributes for the command, return code, stdout, and stderr.
+
+    Raises:
+        CalledProcessError: If the subprocess exits with a non-zero status and `check=True`.
+
+    Examples:
+        >>> process = await run(["ls", "/tmp"])
+        >>> process.stdout
+        "file1.txt\nfile2.txt"
     """
     proc, _input, command = await self._spawn_proc(*command, **kwargs)
     if proc is not None:
@@ -41,10 +59,28 @@ async def run(self, *command, check=False, text=True, **kwargs):
 
 
 async def run_live(self, *command, check=False, text=True, **kwargs):
-    """
-    Simple helper for running a command and iterating through its output line by line in realtime
-        async for line in run_live(["ls", "/tmp"]):
-            log.info(line)
+    """Runs a command asynchronously and iterates through its output line by line in realtime.
+
+    This method is useful for executing a command and capturing its output on-the-fly, as it is generated.
+    If an error occurs during execution, it can optionally raise an error or just log the stderr.
+
+    Args:
+        *command (str): The command to run as separate arguments.
+        check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.
+                                Defaults to False.
+        text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.
+        **kwargs (dict): Additional keyword arguments for the subprocess.
+
+    Yields:
+        str or bytes: The output lines of the command, either as a decoded string (if `text=True`)
+                      or as bytes (if `text=False`).
+
+    Raises:
+        CalledProcessError: If the subprocess exits with a non-zero status and `check=True`.
+
+    Examples:
+        >>> async for line in run_live(["tail", "-f", "/var/log/auth.log"]):
+        ...     log.info(line)
     """
     proc, _input, command = await self._spawn_proc(*command, **kwargs)
     if proc is not None:
@@ -92,6 +128,27 @@ async def run_live(self, *command, check=False, text=True, **kwargs):
 
 
 async def _spawn_proc(self, *command, **kwargs):
+    """Spawns an asynchronous subprocess.
+
+    Prepares the command and associated keyword arguments. If the `input` argument is provided,
+    it checks to ensure that the `stdin` argument is not also provided. Once prepared, it creates
+    and returns the subprocess. If the command executable is not found, it logs a warning and traceback.
+
+    Args:
+        *command (str): The command to run as separate arguments.
+        **kwargs (dict): Additional keyword arguments for the subprocess.
+
+    Raises:
+        ValueError: If both stdin and input arguments are provided.
+
+    Returns:
+        tuple: A tuple containing the created process (or None if creation failed), the input (or None if not provided),
+               and the prepared command (or None if subprocess creation failed).
+
+    Examples:
+        >>> _spawn_proc("ls", "-l", input="data")
+        (<Process ...>, "data", ["ls", "-l"])
+    """
     command, kwargs = self._prepare_command_kwargs(command, kwargs)
     _input = kwargs.pop("input", None)
     if _input is not None:
@@ -110,6 +167,17 @@ async def _spawn_proc(self, *command, **kwargs):
 
 
 async def _write_stdin(proc, _input):
+    """
+    Asynchronously writes input to an active subprocess's stdin.
+
+    This function takes an `_input` parameter, which can be of type str, bytes,
+    list, tuple, or an asynchronous generator. The input is then written line by
+    line to the stdin of the given `proc`.
+
+    Args:
+        proc (subprocess.Popen): An active subprocess object.
+        _input (str, bytes, list, tuple, async generator): The data to write to stdin.
+    """
     if _input is not None:
         if isinstance(_input, (str, bytes)):
             _input = [_input]
@@ -124,6 +192,28 @@ async def _write_stdin(proc, _input):
 
 
 def _prepare_command_kwargs(self, command, kwargs):
+    """
+    Prepare arguments for passing into `asyncio.create_subprocess_exec()`.
+
+    This method modifies the `kwargs` dictionary in place to prepare it for
+    use in the `asyncio.create_subprocess_exec()` method. It sets the default
+    values for keys like 'limit', 'stdout', and 'stderr' if they are not
+    already present. It also handles the case when 'sudo' needs to be run.
+
+    Args:
+        command (list): The command to be run in the subprocess.
+        kwargs (dict): The keyword arguments to be passed to `asyncio.create_subprocess_exec()`.
+
+    Returns:
+        tuple: A tuple containing the modified `command` and `kwargs`.
+
+    Examples:
+        >>> _prepare_command_kwargs(['ls', '-l'], {})
+        (['ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1})
+
+        >>> _prepare_command_kwargs(['ls', '-l'], {'sudo': True})
+        (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
+    """
     # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
     if not "limit" in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100