integrate postman with httpx

bbot/modules/httpx.py

@@ -85,7 +85,8 @@ async def handle_batch(self, *events):
 
         command = [
             "httpx",
-            "-silent",
+            "-debug",
+            # "-silent",
             "-json",
             "-include-response",
             "-threads",

bbot/modules/internal/excavate.py

@@ -269,7 +269,7 @@ class JavascriptExtractor(BaseExtractor):
         "firebase": r"AAAA[A-Za-z0-9_-]{7}:[A-Za-z0-9_-]{140}",
         "google_oauth": r"ya29\.[0-9A-Za-z\-_]+",
         "amazon_aws_access_key_id": r"A[SK]IA[0-9A-Z]{16}",
-        "amazon_mws_auth_toke": r"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+        "amazon_mws_auth_token": r"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
         # "amazon_aws_url": r"s3\.amazonaws.com[/]+|[a-zA-Z0-9_-]*\.s3\.amazonaws.com",
         # "amazon_aws_url2": r"[a-zA-Z0-9-\.\_]+\.s3\.amazonaws\.com",
         # "amazon_aws_url3": r"s3://[a-zA-Z0-9-\.\_]+",

bbot/modules/postman.py

@@ -1,7 +1,7 @@
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
-class postman(BaseModule):
+class postman(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["passive", "subdomain-enum", "safe"]
@@ -17,14 +17,16 @@ class postman(BaseModule):
         "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
     }
 
+    reject_wildcards = False
+
+    # wait until outgoing queue is empty to help avoid rate limits
+    _qsize = 1
+
     async def handle_event(self, event):
-        if "target" in event.tags:
-            query = str(event.data)
-        else:
-            query = self.helpers.parent_domain(event.data).lower()
+        query = self.make_query(event)
         self.verbose(f"Search for any postman workspaces, collections, requests belonging to {query}")
         for url in await self.query(query):
-            self.emit_event(url, "URL_UNVERIFIED", source=event)
+            self.emit_event(url, "URL_UNVERIFIED", source=event, tags="httpx-safe")
 
     async def query(self, query):
         interesting_urls = []

bbot/test/test_step_2/module_tests/test_module_postman.py

@@ -7,7 +7,9 @@ class TestPostman(ModuleTestBase):
         "scope_report_distance": 1,
     }
 
-    async def setup_before_prep(self, module_test):
+    modules_overrides = ["postman", "httpx", "excavate"]
+
+    async def setup_after_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
             json={
@@ -179,31 +181,51 @@ async def setup_before_prep(self, module_test):
             },
         )
 
+        old_emit_event = module_test.module.emit_event
+
+        def new_emit_event(event_data, event_type, **kwargs):
+            if event_data.startswith("https://www.postman.com"):
+                event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888")
+            old_emit_event(event_data, event_type, **kwargs)
+
+        module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event)
+        module_test.scan.helpers.dns.mock_dns({("asdf.blacklanternsecurity.com", "A"): "127.0.0.1"})
+
+        request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862")
+        respond_args = dict(response_data="https://asdf.blacklanternsecurity.com")
+        module_test.set_expect_requests(request_args, respond_args)
+
     def check(self, module_test, events):
         assert any(
-            e.data == "https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events
+            e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events
         ), "Failed to detect workspace"
         assert any(
-            e.data == "https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals"
+            e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals"
             for e in events
         ), "Failed to detect workspace globals"
         assert any(
-            e.data == "https://www.postman.com/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567"
+            e.data == "http://127.0.0.1:8888/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567"
             for e in events
         ), "Failed to detect workspace environment"
         assert any(
-            e.data == "https://www.postman.com/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"
+            e.data == "http://127.0.0.1:8888/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"
             for e in events
         ), "Failed to detect collection"
         assert any(
-            e.data == "https://www.postman.com/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
+            e.data == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
             for e in events
         ), "Failed to detect collection request #1"
         assert any(
-            e.data == "https://www.postman.com/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409"
+            e.data == "http://127.0.0.1:8888/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409"
             for e in events
-        ), "Failed to detect collection request #1"
+        ), "Failed to detect collection request #2"
         assert any(
-            e.data == "https://www.postman.com/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896"
+            e.data == "http://127.0.0.1:8888/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896"
             for e in events
-        ), "Failed to detect collection request #1"
+        ), "Failed to detect collection request #3"
+        assert any(
+            e.type == "HTTP_RESPONSE"
+            and e.data["url"] == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
+            for e in events
+        ), "Failed to emit HTTP_RESPONSE"
+        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"