From 9fedf1d61ba53d4d04212d4bb3043d42886181a8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 13:19:04 -0500 Subject: [PATCH 1/4] support filesystem, mobile app targets --- README.md | 5 ++++ bbot/core/event/base.py | 23 +++++++++++++++++- bbot/scanner/target.py | 16 ++++++++++++- bbot/test/bbot_fixtures.py | 14 +++++------ bbot/test/test_step_1/test_events.py | 36 ++++++++++++++++++++++++++++ docs/scanning/index.md | 5 ++++ 6 files changed, 90 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bd4e9d461a..ee5790ee5d 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,11 @@ Targets can be any of the following: - `IP_RANGE` (`1.2.3.0/24`) - `OPEN_TCP_PORT` (`192.168.0.1:80`) - `URL` (`https://www.evilcorp.com`) +- `EMAIL_ADDRESS` (`bob@evilcorp.com`) +- `ORG_STUB` (`ORG:evilcorp`) +- `USER_STUB` (`USER:bobsmith`) +- `FILESYSTEM` (`FILESYSTEM:/tmp/asdf`) +- `MOBILE_APP` (`MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app`) For more information, see [Targets](https://www.blacklanternsecurity.com/bbot/Stable/scanning/#targets-t). To learn how BBOT handles scope, see [Scope](https://www.blacklanternsecurity.com/bbot/Stable/scanning/#scope). diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 32d6f7a3a1..f669c65ff7 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -14,8 +14,8 @@ from typing import Optional from contextlib import suppress from radixtarget import RadixTarget -from urllib.parse import urljoin, parse_qs from pydantic import BaseModel, field_validator +from urllib.parse import urlparse, urljoin, parse_qs from .helpers import * @@ -1584,6 +1584,27 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent): class MOBILE_APP(DictEvent): _always_emit = True + def _sanitize_data(self, data): + if isinstance(data, str): + data = {"url": data} + if "url" not in data: + raise ValidationError("url is required for MOBILE_APP events") + url = data["url"] + # parse URL + try: + self.parsed_url = urlparse(url) + except Exception as e: + raise ValidationError(f"Error parsing URL {url}: {e}") + if not "id" in data: + # extract "id" getparam + params = parse_qs(self.parsed_url.query) + try: + _id = params["id"][0] + except Exception: + raise ValidationError("id is required for MOBILE_APP events") + data["id"] = _id + return data + def _pretty_string(self): return self.data["url"] diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index f86b0de15b..bdd9edd107 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -95,9 +95,9 @@ def add(self, targets): else: event = self.make_event(target) if event: + self.inputs.add(target) _events = [event] for event in _events: - self.inputs.add(event.data) events.add(event) # sort by host size to ensure consistency @@ -140,6 +140,20 @@ def handle_username(self, match): return [username_event] return [] + @special_target_type(r"^(?:FILESYSTEM|FILE|FOLDER|DIR|PATH):(.*)") + def handle_filesystem(self, match): + filesystem_event = self.make_event({"path": match.group(1)}, event_type="FILESYSTEM") + if filesystem_event: + return [filesystem_event] + return [] + + @special_target_type(r"^(?:MOBILE_APP|APK|IPA|APP):(.*)") + def handle_mobile_app(self, match): + mobile_app_event = self.make_event({"url": match.group(1)}, event_type="MOBILE_APP") + if mobile_app_event: + return [mobile_app_event] + return [] + def get(self, event, single=True, **kwargs): results = super().get(event, **kwargs) if results and single: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 070df6e9a3..86bc83433f 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,12 +224,12 @@ class bbot_events: return bbot_events -@pytest.fixture(scope="session", autouse=True) -def install_all_python_deps(): - deps_pip = set() - for module in DEFAULT_PRESET.module_loader.preloaded().values(): - deps_pip.update(set(module.get("deps", {}).get("pip", []))) +# @pytest.fixture(scope="session", autouse=True) +# def install_all_python_deps(): +# deps_pip = set() +# for module in DEFAULT_PRESET.module_loader.preloaded().values(): +# deps_pip.update(set(module.get("deps", {}).get("pip", []))) - constraint_file = tempwordlist(get_python_constraints()) +# constraint_file = tempwordlist(get_python_constraints()) - subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) +# subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 195f08ea89..abfaad8f06 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -979,6 +979,42 @@ def test_event_magic(): zip_file.unlink() +@pytest.mark.asyncio +async def test_mobile_app(): + scan = Scanner() + with pytest.raises(ValidationError): + scan.make_event("com.evilcorp.app", "MOBILE_APP", parent=scan.root_event) + with pytest.raises(ValidationError): + scan.make_event({"id": "com.evilcorp.app"}, "MOBILE_APP", parent=scan.root_event) + with pytest.raises(ValidationError): + scan.make_event({"url": "https://play.google.com/store/apps/details"}, "MOBILE_APP", parent=scan.root_event) + mobile_app = scan.make_event( + {"url": "https://play.google.com/store/apps/details?id=com.evilcorp.app"}, "MOBILE_APP", parent=scan.root_event + ) + assert sorted(mobile_app.data.items()) == [ + ("id", "com.evilcorp.app"), + ("url", "https://play.google.com/store/apps/details?id=com.evilcorp.app"), + ] + + scan = Scanner("MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app") + events = [e async for e in scan.async_start()] + assert len(events) == 3 + assert events[1].type == "MOBILE_APP" + assert sorted(events[1].data.items()) == [ + ("id", "com.evilcorp.app"), + ("url", "https://play.google.com/store/apps/details?id=com.evilcorp.app"), + ] + + +@pytest.mark.asyncio +async def test_filesystem(): + scan = Scanner("FILESYSTEM:/tmp/asdf") + events = [e async for e in scan.async_start()] + assert len(events) == 3 + assert events[1].type == "FILESYSTEM" + assert events[1].data == {"path": "/tmp/asdf"} + + def test_event_hashing(): scan = Scanner("example.com") url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event) diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 357dc5294d..b947319c45 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -22,6 +22,11 @@ Targets declare what's in-scope, and seed a scan with initial data. BBOT accepts - `IP_RANGE` (`1.2.3.0/24`) - `OPEN_TCP_PORT` (`192.168.0.1:80`) - `URL` (`https://www.evilcorp.com`) +- `EMAIL_ADDRESS` (`bob@evilcorp.com`) +- `ORG_STUB` (`ORG:evilcorp`) +- `USER_STUB` (`USER:bobsmith`) +- `FILESYSTEM` (`FILESYSTEM:/tmp/asdf`) +- `MOBILE_APP` (`MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app`) Note that BBOT only discriminates down to the host level. This means, for example, if you specify a URL `https://www.evilcorp.com` as the target, the scan will be *seeded* with that URL, but the scope of the scan will be the entire host, `www.evilcorp.com`. Other ports/URLs on that same host may also be scanned. From 9366687ef661b3870ea6c14f7196368b4224ca74 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 14:24:19 -0500 Subject: [PATCH 2/4] test troubleshooting --- bbot/test/test_step_1/test_events.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index abfaad8f06..2de0641e54 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -999,8 +999,9 @@ async def test_mobile_app(): scan = Scanner("MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app") events = [e async for e in scan.async_start()] assert len(events) == 3 - assert events[1].type == "MOBILE_APP" - assert sorted(events[1].data.items()) == [ + mobile_app_event = [e for e in events if e.type == "MOBILE_APP"][0] + assert mobile_app_event.type == "MOBILE_APP" + assert sorted(mobile_app_event.data.items()) == [ ("id", "com.evilcorp.app"), ("url", "https://play.google.com/store/apps/details?id=com.evilcorp.app"), ] From e47b043963214ee21ea65133f3bbca0d90954017 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 15:24:44 -0500 Subject: [PATCH 3/4] fix preset testsg --- bbot/test/test_step_1/test_presets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 5b1564f12c..be31b38673 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -272,13 +272,13 @@ def test_preset_scope(): } assert preset_whitelist_baked.to_dict(include_target=True) == { "target": ["evilcorp.org"], - "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"], + "whitelist": ["1.2.3.4/24", "http://evilcorp.net"], "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"], "config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}}, } assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == { "target": ["evilcorp.org"], - "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"], + "whitelist": ["1.2.3.4/24", "http://evilcorp.net"], "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"], "config": {"modules": {"secretsdb": {"otherthing": "asdf"}}}, } From cab7aaa6d70d2e8aecd548c8978d22b63ca2477c Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 16:39:49 -0500 Subject: [PATCH 4/4] fix tests --- bbot/test/bbot_fixtures.py | 14 +++++++------- bbot/test/test_step_1/test_events.py | 6 ++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 86bc83433f..070df6e9a3 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,12 +224,12 @@ class bbot_events: return bbot_events -# @pytest.fixture(scope="session", autouse=True) -# def install_all_python_deps(): -# deps_pip = set() -# for module in DEFAULT_PRESET.module_loader.preloaded().values(): -# deps_pip.update(set(module.get("deps", {}).get("pip", []))) +@pytest.fixture(scope="session", autouse=True) +def install_all_python_deps(): + deps_pip = set() + for module in DEFAULT_PRESET.module_loader.preloaded().values(): + deps_pip.update(set(module.get("deps", {}).get("pip", []))) -# constraint_file = tempwordlist(get_python_constraints()) + constraint_file = tempwordlist(get_python_constraints()) -# subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) + subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 2de0641e54..c4ecfbd161 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -1012,8 +1012,10 @@ async def test_filesystem(): scan = Scanner("FILESYSTEM:/tmp/asdf") events = [e async for e in scan.async_start()] assert len(events) == 3 - assert events[1].type == "FILESYSTEM" - assert events[1].data == {"path": "/tmp/asdf"} + filesystem_events = [e for e in events if e.type == "FILESYSTEM"] + assert len(filesystem_events) == 1 + assert filesystem_events[0].type == "FILESYSTEM" + assert filesystem_events[0].data == {"path": "/tmp/asdf"} def test_event_hashing():