From df1956b3b29dbe4d9937f6c60b1be2465b6ae382 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 22 Dec 2023 23:24:00 -0500 Subject: [PATCH] integrate postman with httpx --- bbot/modules/httpx.py | 3 +- bbot/modules/internal/excavate.py | 2 +- bbot/modules/postman.py | 16 +++---- .../module_tests/test_module_postman.py | 42 ++++++++++++++----- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 3c78f95060..5af2d31bc8 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -85,7 +85,8 @@ async def handle_batch(self, *events): command = [ "httpx", - "-silent", + "-debug", + # "-silent", "-json", "-include-response", "-threads", diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index e27e6d228a..c58b01de63 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -269,7 +269,7 @@ class JavascriptExtractor(BaseExtractor): "firebase": r"AAAA[A-Za-z0-9_-]{7}:[A-Za-z0-9_-]{140}", "google_oauth": r"ya29\.[0-9A-Za-z\-_]+", "amazon_aws_access_key_id": r"A[SK]IA[0-9A-Z]{16}", - "amazon_mws_auth_toke": r"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "amazon_mws_auth_token": r"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", # "amazon_aws_url": r"s3\.amazonaws.com[/]+|[a-zA-Z0-9_-]*\.s3\.amazonaws.com", # "amazon_aws_url2": r"[a-zA-Z0-9-\.\_]+\.s3\.amazonaws\.com", # "amazon_aws_url3": r"s3://[a-zA-Z0-9-\.\_]+", diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 3c485ab523..26d7f8df89 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -1,7 +1,7 @@ -from bbot.modules.base import BaseModule +from bbot.modules.templates.subdomain_enum import subdomain_enum -class postman(BaseModule): +class postman(subdomain_enum): watched_events = ["DNS_NAME"] produced_events = ["URL_UNVERIFIED"] flags = ["passive", "subdomain-enum", "safe"] @@ -17,14 +17,16 @@ class postman(BaseModule): "Referer": "https://www.postman.com/search?q=&scope=public&type=all", } + reject_wildcards = False + + # wait until outgoing queue is empty to help avoid rate limits + _qsize = 1 + async def handle_event(self, event): - if "target" in event.tags: - query = str(event.data) - else: - query = self.helpers.parent_domain(event.data).lower() + query = self.make_query(event) self.verbose(f"Search for any postman workspaces, collections, requests belonging to {query}") for url in await self.query(query): - self.emit_event(url, "URL_UNVERIFIED", source=event) + self.emit_event(url, "URL_UNVERIFIED", source=event, tags="httpx-safe") async def query(self, query): interesting_urls = [] diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index 91c590198e..2879b4838a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -7,7 +7,9 @@ class TestPostman(ModuleTestBase): "scope_report_distance": 1, } - async def setup_before_prep(self, module_test): + modules_overrides = ["postman", "httpx", "excavate"] + + async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( url="https://www.postman.com/_api/ws/proxy", json={ @@ -179,31 +181,51 @@ async def setup_before_prep(self, module_test): }, ) + old_emit_event = module_test.module.emit_event + + def new_emit_event(event_data, event_type, **kwargs): + if event_data.startswith("https://www.postman.com"): + event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888") + old_emit_event(event_data, event_type, **kwargs) + + module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event) + module_test.scan.helpers.dns.mock_dns({("asdf.blacklanternsecurity.com", "A"): "127.0.0.1"}) + + request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862") + respond_args = dict(response_data="https://asdf.blacklanternsecurity.com") + module_test.set_expect_requests(request_args, respond_args) + def check(self, module_test, events): assert any( - e.data == "https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events + e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events ), "Failed to detect workspace" assert any( - e.data == "https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals" + e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals" for e in events ), "Failed to detect workspace globals" assert any( - e.data == "https://www.postman.com/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567" + e.data == "http://127.0.0.1:8888/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567" for e in events ), "Failed to detect workspace environment" assert any( - e.data == "https://www.postman.com/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205" + e.data == "http://127.0.0.1:8888/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205" for e in events ), "Failed to detect collection" assert any( - e.data == "https://www.postman.com/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862" + e.data == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862" for e in events ), "Failed to detect collection request #1" assert any( - e.data == "https://www.postman.com/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409" + e.data == "http://127.0.0.1:8888/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409" for e in events - ), "Failed to detect collection request #1" + ), "Failed to detect collection request #2" assert any( - e.data == "https://www.postman.com/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896" + e.data == "http://127.0.0.1:8888/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896" for e in events - ), "Failed to detect collection request #1" + ), "Failed to detect collection request #3" + assert any( + e.type == "HTTP_RESPONSE" + and e.data["url"] == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862" + for e in events + ), "Failed to emit HTTP_RESPONSE" + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"