steady work

bbot/core/helpers/depsinstaller/installer.py

@@ -14,7 +14,7 @@
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform, rm_at_exit
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -173,10 +173,7 @@ async def pip_install(self, packages, constraints=None):
 
         # if no custom constraints are provided, use the constraints of the currently installed version of bbot
         if constraints is not None:
-            from importlib.metadata import distribution
-
-            dist = distribution("bbot")
-            constraints = [str(r) for r in dist.requires]
+            constraints = get_python_constraints()
 
         constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
         command.append("--constraint")

bbot/core/helpers/misc.py

@@ -2807,3 +2807,21 @@ def safe_format(s, **kwargs):
     Format string while ignoring unused keys (prevents KeyError)
     """
     return s.format_map(SafeDict(kwargs))
+
+
+def get_python_constraints():
+    req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE)
+
+    def clean_requirement(req_string):
+        # Extract package name and version constraints from format like "package (>=1.0,<2.0)"
+        match = req_regex.match(req_string)
+        if match:
+            name, constraints = match.groups()
+            return f"{name.strip()}{constraints}"
+
+        return req_string
+
+    from importlib.metadata import distribution
+
+    dist = distribution("bbot")
+    return [clean_requirement(r) for r in dist.requires]

bbot/test/bbot_fixtures.py

@@ -15,8 +15,8 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
 log = logging.getLogger(f"bbot.test.fixtures")
@@ -230,10 +230,6 @@ def install_all_python_deps():
     for module in DEFAULT_PRESET.module_loader.preloaded().values():
         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
 
-    from importlib.metadata import distribution
-
-    dist = distribution("bbot")
-    constraints = [str(r) for r in dist.requires]
-    constraint_file = tempwordlist(constraints)
+    constraint_file = tempwordlist(get_python_constraints())
 
     subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))

bbot/test/conftest.py

@@ -95,9 +95,21 @@ def bbot_httpserver_ssl():
     server.clear()
 
 
-@pytest.fixture
-def non_mocked_hosts() -> list:
-    return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+def should_mock(request):
+    return not request.url.host in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+
+
+def pytest_collection_modifyitems(config, items):
+    # make sure all tests have the httpx_mock marker
+    for item in items:
+        # if "httpx_mock" not in item.keywords:
+        item.add_marker(
+            pytest.mark.httpx_mock(
+                should_mock=should_mock,
+                assert_all_requests_were_expected=False,
+                can_send_already_matched_responses=True,
+            )
+        )
 
 
 @pytest.fixture

bbot/test/test_step_1/test_events.py

@@ -484,7 +484,6 @@ async def test_events(events, helpers):
     json_event = db_event.json()
     assert isinstance(json_event["uuid"], str)
     assert json_event["uuid"] == str(db_event.uuid)
-    print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}")
     assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"

bbot/test/test_step_1/test_web.py

@@ -471,6 +471,9 @@ async def test_web_cookies(bbot_scanner, httpx_mock):
     # but that they're not sent in the response
     with pytest.raises(httpx.TimeoutException):
         r = await client2.get(url="http://www2.evilcorp.com/cookies/test")
+    # make sure cookies are sent
+    r = await client2.get(url="http://www2.evilcorp.com/cookies/test", cookies={"wats": "fdsa"})
+    assert r.status_code == 200
     # make sure we can manually send cookies
     httpx_mock.add_response(url="http://www2.evilcorp.com/cookies/test2", match_headers={"Cookie": "fdsa=wats"})
     r = await client2.get(url="http://www2.evilcorp.com/cookies/test2", cookies={"fdsa": "wats"})

bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py

@@ -92,9 +92,6 @@ def check(self, module_test, events):
         dnn_installwizard_privesc_detection = False
 
         for e in events:
-            print(e)
-            print(e.type)
-
             if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]:
                 dnn_technology_detection = True
 
@@ -170,9 +167,6 @@ def check(self, module_test, events):
         dnn_dnnimagehandler_blindssrf = False
 
         for e in events:
-
-            print(e)
-            print(e.type)
             if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]:
                 dnn_technology_detection = True
 

docs/scanning/output.md

@@ -90,10 +90,11 @@ mail.evilcorp.com
 
 BBOT supports output via webhooks to `discord`, `slack`, and `teams`. To use them, you must specify a webhook URL either in the config:
 
-```yaml title="~/.bbot/config/bbot.yml"
-modules:
-  discord:
-    webhook_url: https://discord.com/api/webhooks/1234/deadbeef
+```yaml title="discord_preset.yml"
+config:
+  modules:
+    discord:
+      webhook_url: https://discord.com/api/webhooks/1234/deadbeef
 ```
 
 ...or on the command line:
@@ -103,13 +104,14 @@ bbot -t evilcorp.com -om discord -c modules.discord.webhook_url=https://discord.
 
 By default, only `VULNERABILITY` and `FINDING` events are sent, but this can be customized by setting `event_types` in the config like so:
 
-```yaml title="~/.bbot/config/bbot.yml"
-modules:
-  discord:
-    event_types:
-      - VULNERABILITY
-      - FINDING
-      - STORAGE_BUCKET
+```yaml title="discord_preset.yml"
+config:
+  modules:
+    discord:
+      event_types:
+        - VULNERABILITY
+        - FINDING
+        - STORAGE_BUCKET
 ```
 
 ...or on the command line:
@@ -120,10 +122,11 @@ bbot -t evilcorp.com -om discord -c modules.discord.event_types=["STORAGE_BUCKET
 You can also filter on the severity of `VULNERABILITY` events by setting `min_severity`:
 
 
-```yaml title="~/.bbot/config/bbot.yml"
-modules:
-  discord:
-    min_severity: HIGH
+```yaml title="discord_preset.yml"
+config:
+  modules:
+    discord:
+      min_severity: HIGH
 ```
 
 ### HTTP
@@ -137,16 +140,42 @@ bbot -t evilcorp.com -om http -c modules.http.url=http://localhost:8000
 
 You can customize the HTTP method if needed. Authentication is also supported:
 
-```yaml title="~/.bbot/config/bbot.yml"
-modules:
-  http:
-    url: https://localhost:8000
-    method: PUT
-    # Authorization: Bearer
-    bearer: <bearer_token>
-    # OR
-    username: bob
-    password: P@ssw0rd
+```yaml title="http_preset.yml"
+config:
+  modules:
+    http:
+      url: https://localhost:8000
+      method: PUT
+      # Authorization: Bearer
+      bearer: <bearer_token>
+      # OR
+      username: bob
+      password: P@ssw0rd
+```
+
+### Elasticsearch
+
+When outputting to Elastic, use the `http` output module with the following settings (replace `<your_index>` with your desired index, e.g. `bbot`):
+
+```bash
+# send scan results directly to elasticsearch
+bbot -t evilcorp.com -om http -c \
+  modules.http.url=http://localhost:8000/<your_index>/_doc \
+  modules.http.siem_friendly=true \
+  modules.http.username=elastic \
+  modules.http.password=changeme
+```
+
+Alternatively, via a preset:
+
+```yaml title="elastic_preset.yml"
+config:
+  modules:
+    http:
+      url: http://localhost:8000/<your_index>/_doc
+      siem_friendly: true
+      username: elastic
+      password: changeme
 ```
 
 ### Splunk
@@ -155,17 +184,18 @@ The `splunk` output module sends [events](events.md) in JSON format to a desired
 
 You can customize this output with the following config options:
 
-```yaml title="~/.bbot/config/bbot.yml"
-modules:
-  splunk:
-    # The full URL with the URI `/services/collector/event`
-    url: https://localhost:8088/services/collector/event
-    # Generated from splunk webui
-    hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-    # Defaults to `main` if not set
-    index: my-specific-index
-    # Defaults to `bbot` if not set
-    source: /my/source.json
+```yaml title="splunk_preset.yml"
+config:
+  modules:
+    splunk:
+      # The full URL with the URI `/services/collector/event`
+      url: https://localhost:8088/services/collector/event
+      # Generated from splunk webui
+      hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+      # Defaults to `main` if not set
+      index: my-specific-index
+      # Defaults to `bbot` if not set
+      source: /my/source.json
 ```
 
 ### Asset Inventory

docs/scanning/tips_and_tricks.md

@@ -79,13 +79,13 @@ bbot -t evilcorp.com -f subdomain-enum -c spider.yml
 
 ### Ingesting BBOT Data Into SIEM (Elastic, Splunk)
 
-If your goal is to feed BBOT data into a SIEM such as Elastic, be sure to enable this option when scanning:
+If your goal is to run a BBOT scan and later feed its data into a SIEM such as Elastic, be sure to enable this option when scanning:
 
 ```bash
 bbot -t evilcorp.com -c modules.json.siem_friendly=true
 ```
 
-This nests the event's `.data` beneath its event type like so:
+This ensures the `.data` event attribute is always the same type (a dictionary), by nesting it like so:
 ```json
 {
   "type": "DNS_NAME",