From f1a7b182a22f7d6eda9b382e49a37bfbf786cde8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 4 Nov 2024 17:42:55 -0500 Subject: [PATCH] steady work --- bbot/core/helpers/depsinstaller/installer.py | 7 +- bbot/core/helpers/misc.py | 18 ++++ bbot/test/bbot_fixtures.py | 8 +- bbot/test/conftest.py | 18 +++- bbot/test/test_step_1/test_events.py | 1 - bbot/test/test_step_1/test_web.py | 3 + .../module_tests/test_module_dotnetnuke.py | 6 -- docs/scanning/output.md | 102 +++++++++++------- docs/scanning/tips_and_tricks.md | 4 +- 9 files changed, 108 insertions(+), 59 deletions(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 771c8f8d06..6a6f602115 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -14,7 +14,7 @@ from ansible_runner.interface import run from subprocess import CalledProcessError -from ..misc import can_sudo_without_password, os_platform, rm_at_exit +from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints log = logging.getLogger("bbot.core.helpers.depsinstaller") @@ -173,10 +173,7 @@ async def pip_install(self, packages, constraints=None): # if no custom constraints are provided, use the constraints of the currently installed version of bbot if constraints is not None: - from importlib.metadata import distribution - - dist = distribution("bbot") - constraints = [str(r) for r in dist.requires] + constraints = get_python_constraints() constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False) command.append("--constraint") diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index c493bd4d37..c416e54f9f 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2807,3 +2807,21 @@ def safe_format(s, **kwargs): Format string while ignoring unused keys (prevents KeyError) """ return s.format_map(SafeDict(kwargs)) + + +def get_python_constraints(): + req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE) + + def clean_requirement(req_string): + # Extract package name and version constraints from format like "package (>=1.0,<2.0)" + match = req_regex.match(req_string) + if match: + name, constraints = match.groups() + return f"{name.strip()}{constraints}" + + return req_string + + from importlib.metadata import distribution + + dist = distribution("bbot") + return [clean_requirement(r) for r in dist.requires] diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index fa45bc4766..e1e3aa1b8b 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -15,8 +15,8 @@ from bbot.errors import * # noqa: F401 from bbot.core import CORE from bbot.scanner import Preset -from bbot.core.helpers.misc import mkdir, rand_string from bbot.core.helpers.async_helpers import get_event_loop +from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints log = logging.getLogger(f"bbot.test.fixtures") @@ -230,10 +230,6 @@ def install_all_python_deps(): for module in DEFAULT_PRESET.module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) - from importlib.metadata import distribution - - dist = distribution("bbot") - constraints = [str(r) for r in dist.requires] - constraint_file = tempwordlist(constraints) + constraint_file = tempwordlist(get_python_constraints()) subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index e82333552b..345f0499bc 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -95,9 +95,21 @@ def bbot_httpserver_ssl(): server.clear() -@pytest.fixture -def non_mocked_hosts() -> list: - return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers +def should_mock(request): + return not request.url.host in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers + + +def pytest_collection_modifyitems(config, items): + # make sure all tests have the httpx_mock marker + for item in items: + # if "httpx_mock" not in item.keywords: + item.add_marker( + pytest.mark.httpx_mock( + should_mock=should_mock, + assert_all_requests_were_expected=False, + can_send_already_matched_responses=True, + ) + ) @pytest.fixture diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 1ebb38fea3..46ddbb1f1c 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -484,7 +484,6 @@ async def test_events(events, helpers): json_event = db_event.json() assert isinstance(json_event["uuid"], str) assert json_event["uuid"] == str(db_event.uuid) - print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}") assert json_event["parent_uuid"] == str(scan.root_event.uuid) assert json_event["scope_distance"] == 1 assert json_event["data"] == "evilcorp.com:80" diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index 4d5654c86b..0b3011d572 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -471,6 +471,9 @@ async def test_web_cookies(bbot_scanner, httpx_mock): # but that they're not sent in the response with pytest.raises(httpx.TimeoutException): r = await client2.get(url="http://www2.evilcorp.com/cookies/test") + # make sure cookies are sent + r = await client2.get(url="http://www2.evilcorp.com/cookies/test", cookies={"wats": "fdsa"}) + assert r.status_code == 200 # make sure we can manually send cookies httpx_mock.add_response(url="http://www2.evilcorp.com/cookies/test2", match_headers={"Cookie": "fdsa=wats"}) r = await client2.get(url="http://www2.evilcorp.com/cookies/test2", cookies={"fdsa": "wats"}) diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py index de78ad50b9..2916c527af 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py @@ -92,9 +92,6 @@ def check(self, module_test, events): dnn_installwizard_privesc_detection = False for e in events: - print(e) - print(e.type) - if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]: dnn_technology_detection = True @@ -170,9 +167,6 @@ def check(self, module_test, events): dnn_dnnimagehandler_blindssrf = False for e in events: - - print(e) - print(e.type) if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]: dnn_technology_detection = True diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 7efdf48620..b96008b13a 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -90,10 +90,11 @@ mail.evilcorp.com BBOT supports output via webhooks to `discord`, `slack`, and `teams`. To use them, you must specify a webhook URL either in the config: -```yaml title="~/.bbot/config/bbot.yml" -modules: - discord: - webhook_url: https://discord.com/api/webhooks/1234/deadbeef +```yaml title="discord_preset.yml" +config: + modules: + discord: + webhook_url: https://discord.com/api/webhooks/1234/deadbeef ``` ...or on the command line: @@ -103,13 +104,14 @@ bbot -t evilcorp.com -om discord -c modules.discord.webhook_url=https://discord. By default, only `VULNERABILITY` and `FINDING` events are sent, but this can be customized by setting `event_types` in the config like so: -```yaml title="~/.bbot/config/bbot.yml" -modules: - discord: - event_types: - - VULNERABILITY - - FINDING - - STORAGE_BUCKET +```yaml title="discord_preset.yml" +config: + modules: + discord: + event_types: + - VULNERABILITY + - FINDING + - STORAGE_BUCKET ``` ...or on the command line: @@ -120,10 +122,11 @@ bbot -t evilcorp.com -om discord -c modules.discord.event_types=["STORAGE_BUCKET You can also filter on the severity of `VULNERABILITY` events by setting `min_severity`: -```yaml title="~/.bbot/config/bbot.yml" -modules: - discord: - min_severity: HIGH +```yaml title="discord_preset.yml" +config: + modules: + discord: + min_severity: HIGH ``` ### HTTP @@ -137,16 +140,42 @@ bbot -t evilcorp.com -om http -c modules.http.url=http://localhost:8000 You can customize the HTTP method if needed. Authentication is also supported: -```yaml title="~/.bbot/config/bbot.yml" -modules: - http: - url: https://localhost:8000 - method: PUT - # Authorization: Bearer - bearer: - # OR - username: bob - password: P@ssw0rd +```yaml title="http_preset.yml" +config: + modules: + http: + url: https://localhost:8000 + method: PUT + # Authorization: Bearer + bearer: + # OR + username: bob + password: P@ssw0rd +``` + +### Elasticsearch + +When outputting to Elastic, use the `http` output module with the following settings (replace `` with your desired index, e.g. `bbot`): + +```bash +# send scan results directly to elasticsearch +bbot -t evilcorp.com -om http -c \ + modules.http.url=http://localhost:8000//_doc \ + modules.http.siem_friendly=true \ + modules.http.username=elastic \ + modules.http.password=changeme +``` + +Alternatively, via a preset: + +```yaml title="elastic_preset.yml" +config: + modules: + http: + url: http://localhost:8000//_doc + siem_friendly: true + username: elastic + password: changeme ``` ### Splunk @@ -155,17 +184,18 @@ The `splunk` output module sends [events](events.md) in JSON format to a desired You can customize this output with the following config options: -```yaml title="~/.bbot/config/bbot.yml" -modules: - splunk: - # The full URL with the URI `/services/collector/event` - url: https://localhost:8088/services/collector/event - # Generated from splunk webui - hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - # Defaults to `main` if not set - index: my-specific-index - # Defaults to `bbot` if not set - source: /my/source.json +```yaml title="splunk_preset.yml" +config: + modules: + splunk: + # The full URL with the URI `/services/collector/event` + url: https://localhost:8088/services/collector/event + # Generated from splunk webui + hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + # Defaults to `main` if not set + index: my-specific-index + # Defaults to `bbot` if not set + source: /my/source.json ``` ### Asset Inventory diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md index 32b55448f1..e6d11474ea 100644 --- a/docs/scanning/tips_and_tricks.md +++ b/docs/scanning/tips_and_tricks.md @@ -79,13 +79,13 @@ bbot -t evilcorp.com -f subdomain-enum -c spider.yml ### Ingesting BBOT Data Into SIEM (Elastic, Splunk) -If your goal is to feed BBOT data into a SIEM such as Elastic, be sure to enable this option when scanning: +If your goal is to run a BBOT scan and later feed its data into a SIEM such as Elastic, be sure to enable this option when scanning: ```bash bbot -t evilcorp.com -c modules.json.siem_friendly=true ``` -This nests the event's `.data` beneath its event type like so: +This ensures the `.data` event attribute is always the same type (a dictionary), by nesting it like so: ```json { "type": "DNS_NAME",