fix conflict

README.md

@@ -226,8 +226,6 @@ config:
     baddns:
       enable_references: True
 
-
-
 ```
 
 </details>

bbot/cli.py

@@ -260,9 +260,7 @@ async def akeyboard_listen():
     finally:
         # save word cloud
         with suppress(BaseException):
-            save_success, filename = scan.helpers.word_cloud.save()
-            if save_success:
-                log_to_stderr(f"Saved word cloud ({len(scan.helpers.word_cloud):,} words) to {filename}")
+            scan.helpers.word_cloud.save()
         # remove output directory if empty
         with suppress(BaseException):
             scan.home.rmdir()

bbot/modules/extractous.py

@@ -112,7 +112,7 @@ def extract_text(file_path):
         result = ""
         buffer = reader.read(4096)
         while len(buffer) > 0:
-            result += buffer.decode("utf-8")
+            result += buffer.decode("utf-8", errors="ignore")
             buffer = reader.read(4096)
 
         return result.strip()

bbot/modules/httpx.py

@@ -1,5 +1,5 @@
 import re
-import json
+import orjson
 import tempfile
 import subprocess
 from pathlib import Path
@@ -142,11 +142,11 @@ async def handle_batch(self, *events):
         proxy = self.scan.http_proxy
         if proxy:
             command += ["-http-proxy", proxy]
-        async for line in self.run_process_live(command, input=list(stdin), stderr=subprocess.DEVNULL):
+        async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL):
             try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
+                j = await self.helpers.run_in_executor(orjson.loads, line)
+            except orjson.JSONDecodeError:
+                self.warning(f"httpx failed to decode line: {line}")
                 continue
 
             url = j.get("url", "")

bbot/modules/internal/cloudcheck.py

@@ -72,9 +72,10 @@ async def handle_event(self, event, **kwargs):
                 base_kwargs["event_type"] = event_type
                 for sig in sigs:
                     matches = []
-                    if event.type == "HTTP_RESPONSE":
-                        matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
-                    elif event.type.startswith("DNS_NAME"):
+                    # TODO: convert this to an excavate YARA hook
+                    # if event.type == "HTTP_RESPONSE":
+                    #     matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
+                    if event.type.startswith("DNS_NAME"):
                         for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:

bbot/modules/output/mysql.py

@@ -3,7 +3,11 @@
 
 class MySQL(SQLTemplate):
     watched_events = ["*"]
-    meta = {"description": "Output scan data to a MySQL database", "created_date": "2024-11-13", "author": "@TheTechromancer"}
+    meta = {
+        "description": "Output scan data to a MySQL database",
+        "created_date": "2024-11-13",
+        "author": "@TheTechromancer",
+    }
     options = {
         "username": "root",
         "password": "bbotislife",

bbot/modules/trufflehog.py

@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.84.2",
+        "version": "3.87.0",
         "config": "",
         "only_verified": True,
         "concurrency": 8,

bbot/scanner/preset/args.py

@@ -135,14 +135,17 @@ def preset_from_args(self):
             args_preset.core.merge_custom({"modules": {"stdout": {"event_types": self.parsed.event_types}}})
 
         # dependencies
+        deps_config = args_preset.core.custom_config.get("deps", {})
         if self.parsed.retry_deps:
-            args_preset.core.custom_config["deps_behavior"] = "retry_failed"
+            deps_config["behavior"] = "retry_failed"
         elif self.parsed.force_deps:
-            args_preset.core.custom_config["deps_behavior"] = "force_install"
+            deps_config["behavior"] = "force_install"
         elif self.parsed.no_deps:
-            args_preset.core.custom_config["deps_behavior"] = "disable"
+            deps_config["behavior"] = "disable"
         elif self.parsed.ignore_failed_deps:
-            args_preset.core.custom_config["deps_behavior"] = "ignore_failed"
+            deps_config["behavior"] = "ignore_failed"
+        if deps_config:
+            args_preset.core.merge_custom({"deps": deps_config})
 
         # other scan options
         if self.parsed.name is not None:
@@ -295,6 +298,12 @@ def create_parser(self, *args, **kwargs):
         )
 
         output = p.add_argument_group(title="Output")
+        output.add_argument(
+            "-o",
+            "--output-dir",
+            help="Directory to output scan results",
+            metavar="DIR",
+        )
         output.add_argument(
             "-om",
             "--output-modules",
@@ -304,12 +313,6 @@ def create_parser(self, *args, **kwargs):
             metavar="MODULE",
         )
         output.add_argument("-lo", "--list-output-modules", action="store_true", help="List available output modules")
-        output.add_argument(
-            "-o",
-            "--output-dir",
-            help="Directory to output scan results",
-            metavar="DIR",
-        )
         output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format")
         output.add_argument("--brief", "-br", action="store_true", help="Output only the data itself")
         output.add_argument("--event-types", nargs="+", default=[], help="Choose which event types to display")

bbot/scanner/preset/preset.py

@@ -798,7 +798,7 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False)
         # misc scan options
         if self.scan_name:
             preset_dict["scan_name"] = self.scan_name
-        if self.scan_name:
+        if self.scan_name and self.output_dir is not None:
             preset_dict["output_dir"] = self.output_dir
 
         # conditions

bbot/scanner/scanner.py

@@ -124,6 +124,7 @@ def __init__(
         self.duration_seconds = None
 
         self._success = False
+        self._scan_finish_status_message = None
 
         if scan_id is not None:
             self.id = str(scan_id)
@@ -425,14 +426,19 @@ async def async_start(self):
 
             self._stop_log_handlers()
 
+            if self._scan_finish_status_message:
+                log_fn = self.hugesuccess
+                if self.status.startswith("ABORT"):
+                    log_fn = self.hugewarning
+                elif not self._success:
+                    log_fn = self.critical
+                log_fn(self._scan_finish_status_message)
+
     async def _mark_finished(self):
-        log_fn = self.hugesuccess
         if self.status == "ABORTING":
             status = "ABORTED"
-            log_fn = self.hugewarning
         elif not self._success:
             status = "FAILED"
-            log_fn = self.critical
         else:
             status = "FINISHED"
 
@@ -441,9 +447,9 @@ async def _mark_finished(self):
         self.duration_seconds = self.duration.total_seconds()
         self.duration_human = self.helpers.human_timedelta(self.duration)
 
-        status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}"
+        self._scan_finish_status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}"
 
-        scan_finish_event = self.finish_event(status_message, status)
+        scan_finish_event = self.finish_event(self._scan_finish_status_message, status)
 
         # queue final scan event with output modules
         output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"]
@@ -457,7 +463,6 @@ async def _mark_finished(self):
             await asyncio.sleep(0.05)
 
         self.status = status
-        log_fn(status_message)
         return scan_finish_event
 
     def _start_modules(self):

bbot/scanner/target.py

@@ -101,7 +101,7 @@ def add(self, targets):
                 events.add(event)
 
         # sort by host size to ensure consistency
-        events = sorted(events, key=lambda e: (0 if not e.host else host_size_key(e.host)))
+        events = sorted(events, key=lambda e: ((0, 0) if not e.host else host_size_key(e.host)))
         for event in events:
             self.events.add(event)
             self._add(event.host, data=event)

bbot/test/test_step_1/test_cli.py

@@ -1,3 +1,5 @@
+import yaml
+
 from ..bbot_fixtures import *
 
 from bbot import cli
@@ -143,6 +145,20 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert len(out.splitlines()) == 1
     assert out.count(".") > 1
 
+    # deps behavior
+    monkeypatch.setattr("sys.argv", ["bbot", "-n", "depstest", "--retry-deps", "--current-preset"])
+    result = await cli._main()
+    assert result is None
+    out, err = capsys.readouterr()
+    print(out)
+    # parse YAML output
+    preset = yaml.safe_load(out)
+    assert preset == {
+        "description": "depstest",
+        "scan_name": "depstest",
+        "config": {"deps": {"behavior": "retry_failed"}},
+    }
+
     # list modules
     monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"])
     result = await cli._main()
@@ -401,7 +417,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
 async def test_cli_customheaders(monkeypatch, caplog, capsys):
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
-    import yaml
 
     # test custom headers
     monkeypatch.setattr(

bbot/test/test_step_1/test_target.py

@@ -2,7 +2,7 @@
 
 
 @pytest.mark.asyncio
-async def test_target(bbot_scanner):
+async def test_target_basic(bbot_scanner):
     from radixtarget import RadixTarget
     from ipaddress import ip_address, ip_network
     from bbot.scanner.target import BBOTTarget, ScanSeeds
@@ -245,6 +245,17 @@ async def test_target(bbot_scanner):
         assert len(events) == 3
         assert {e.type for e in events} == {"SCAN", "USERNAME"}
 
+    # users + orgs + domains
+    scan = bbot_scanner("USER:evilcorp", "ORG:evilcorp", "evilcorp.com")
+    await scan.helpers.dns._mock_dns(
+        {
+            "evilcorp.com": {"A": ["1.2.3.4"]},
+        },
+    )
+    events = [e async for e in scan.async_start()]
+    assert len(events) == 5
+    assert {e.type for e in events} == {"SCAN", "USERNAME", "ORG_STUB", "DNS_NAME"}
+
     # verify hash values
     bbottarget = BBOTTarget(
         "1.2.3.0/24",

bbot/test/test_step_2/module_tests/test_module_cloudcheck.py

@@ -8,7 +8,7 @@ class TestCloudCheck(ModuleTestBase):
     modules_overrides = ["httpx", "excavate", "cloudcheck"]
 
     async def setup_after_prep(self, module_test):
-        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='asdf.s3.amazonaws.com'/>"})
+        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='http://asdf.s3.amazonaws.com'/>"})
 
         scan = Scanner(config={"cloudcheck": True})
         await scan._prep()