From b1bed05ac1a6e6da4f3afa5e6cb9804cd0c3457c Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 18 Sep 2024 18:54:03 +0100 Subject: [PATCH 1/2] Add a new module to download postman workspaces --- bbot/modules/postman.py | 8 - bbot/modules/postman_download.py | 184 +++++++++++ bbot/modules/templates/postman.py | 9 + .../test_module_postman_download.py | 285 ++++++++++++++++++ 4 files changed, 478 insertions(+), 8 deletions(-) create mode 100644 bbot/modules/postman_download.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_postman_download.py diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 982c9ff96..82a3c8b28 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -11,14 +11,6 @@ class postman(postman): "author": "@domwhewell-sage", } - headers = { - "Content-Type": "application/json", - "X-App-Version": "10.18.8-230926-0808", - "X-Entity-Team-Id": "0", - "Origin": "https://www.postman.com", - "Referer": "https://www.postman.com/search?q=&scope=public&type=all", - } - reject_wildcards = False async def handle_event(self, event): diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py new file mode 100644 index 000000000..807f2e9e9 --- /dev/null +++ b/bbot/modules/postman_download.py @@ -0,0 +1,184 @@ +import zipfile +import json +from pathlib import Path +from bbot.modules.templates.postman import postman + + +class postman_download(postman): + watched_events = ["CODE_REPOSITORY"] + produced_events = ["FILESYSTEM"] + flags = ["passive", "subdomain-enum", "safe", "code-enum"] + meta = { + "description": "Download workspaces, collections, requests from Postman", + "created_date": "2024-09-07", + "author": "@domwhewell-sage", + } + options = {"output_folder": "", "api_key": ""} + options_desc = {"output_folder": "Folder to download postman workspaces to"} + scope_distance_modifier = 2 + + async def setup(self): + self.api_key = self.config.get("api_key", "") + self.authorization_headers = {"X-Api-Key": self.api_key} + + output_folder = self.config.get("output_folder") + if output_folder: + self.output_dir = Path(output_folder) / "postman_workspaces" + else: + self.output_dir = self.scan.home / "postman_workspaces" + self.helpers.mkdir(self.output_dir) + return await self.require_api_key() + + async def ping(self): + url = f"{self.api_url}/me" + response = await self.helpers.request(url, headers=self.authorization_headers) + assert getattr(response, "status_code", 0) == 200, response.text + + async def filter_event(self, event): + if event.type == "CODE_REPOSITORY": + if "postman" not in event.tags: + return False, "event is not a postman workspace" + return True + + async def handle_event(self, event): + repo_url = event.data.get("url") + workspace_id = await self.get_workspace_id(repo_url) + if workspace_id: + self.verbose(f"Found workspace ID {workspace_id} for {repo_url}") + workspace_path = await self.download_workspace(workspace_id) + if workspace_path: + self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}") + codebase_event = self.make_event( + {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event + ) + await self.emit_event( + codebase_event, + context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}", + ) + + async def get_workspace_id(self, repo_url): + workspace_id = "" + profile = repo_url.split("/")[-2] + name = repo_url.split("/")[-1] + url = f"{self.base_url}/ws/proxy" + json = { + "service": "workspaces", + "method": "GET", + "path": f"/workspaces?handle={profile}&slug={name}", + } + r = await self.helpers.request(url, method="POST", json=json, headers=self.headers) + if r is None: + return workspace_id + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return workspace_id + data = json.get("data", []) + if len(data) == 1: + workspace_id = data[0]["id"] + return workspace_id + + async def download_workspace(self, id): + zip_path = None + workspace = await self.get_workspace(id) + if workspace: + # Create a folder for the workspace + name = workspace["name"] + folder = self.output_dir / name + self.helpers.mkdir(folder) + zip_path = folder / f"{id}.zip" + + # Main Workspace + self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json") + + # Workspace global variables + self.verbose(f"Downloading globals for workspace {name}") + globals = await self.get_globals(id) + globals_id = globals["id"] + self.add_json_to_zip(zip_path, globals, f"{globals_id}.postman_environment.json") + + # Workspace Environments + workspace_environments = workspace.get("environments", []) + if workspace_environments: + self.verbose(f"Downloading environments for workspace {name}") + for _ in workspace_environments: + environment_id = _["uid"] + environment = await self.get_environment(environment_id) + self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json") + + # Workspace Collections + workspace_collections = workspace.get("collections", []) + if workspace_collections: + self.verbose(f"Downloading collections for workspace {name}") + for _ in workspace_collections: + collection_id = _["uid"] + collection = await self.get_collection(collection_id) + self.add_json_to_zip(zip_path, collection, f"{collection_id}.postman_collection.json") + return zip_path + + async def get_workspace(self, workspace_id): + workspace = {} + workspace_url = f"{self.api_url}/workspaces/{workspace_id}" + r = await self.helpers.request(workspace_url, headers=self.authorization_headers) + if r is None: + return workspace + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return workspace + workspace = json.get("workspace", {}) + return workspace + + async def get_globals(self, workspace_id): + globals = {} + globals_url = f"{self.base_url}/workspace/{workspace_id}/globals" + r = await self.helpers.request(globals_url, headers=self.headers) + if r is None: + return globals + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return globals + globals = json.get("data", {}) + return globals + + async def get_environment(self, environment_id): + environment = {} + environment_url = f"{self.api_url}/environments/{environment_id}" + r = await self.helpers.request(environment_url, headers=self.authorization_headers) + if r is None: + return environment + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return environment + environment = json.get("environment", {}) + return environment + + async def get_collection(self, collection_id): + collection = {} + collection_url = f"{self.api_url}/collections/{collection_id}" + r = await self.helpers.request(collection_url, headers=self.authorization_headers) + if r is None: + return collection + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return collection + collection = json.get("collection", {}) + return collection + + def add_json_to_zip(self, zip_path, data, filename): + with zipfile.ZipFile(zip_path, "a") as zipf: + json_content = json.dumps(data, indent=4) + zipf.writestr(filename, json_content) diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py index ec2f987b0..38cc3d04b 100644 --- a/bbot/modules/templates/postman.py +++ b/bbot/modules/templates/postman.py @@ -8,4 +8,13 @@ class postman(BaseModule): """ base_url = "https://www.postman.com/_api" + api_url = "https://api.getpostman.com" html_url = "https://www.postman.com" + + headers = { + "Content-Type": "application/json", + "X-App-Version": "10.18.8-230926-0808", + "X-Entity-Team-Id": "0", + "Origin": "https://www.postman.com", + "Referer": "https://www.postman.com/search?q=&scope=public&type=all", + } diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py new file mode 100644 index 000000000..83b33f9c5 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_postman_download.py @@ -0,0 +1,285 @@ +from .base import ModuleTestBase + + +class TestPostman_Download(ModuleTestBase): + config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}} + modules_overrides = ["postman", "postman_download", "speculate"] + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/me", + json={ + "user": { + "id": 000000, + "username": "test_key", + "email": "blacklanternsecurity@test.com", + "fullName": "Test Key", + "avatar": "", + "isPublic": True, + "teamId": 0, + "teamDomain": "", + "roles": ["user"], + }, + "operations": [ + {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0}, + {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0}, + {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0}, + {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0}, + {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0}, + {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0}, + {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0}, + {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0}, + ], + }, + ) + + async def setup_after_prep(self, module_test): + await module_test.mock_dns( + {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}} + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}', + json={ + "data": [ + { + "score": 611.41156, + "normalizedScore": 23, + "document": { + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "blacklanternsecurity", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", + "slug": "bbot-public", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "blacklanternsecurity", + "name": "BlackLanternSecurity BBOT [Public]", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", + "privateNetworkFolders": [], + "isPublisherVerified": False, + "publisherType": "team", + "curatedInList": [], + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "", + "popularity": 5, + "isPublic": True, + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "BLS public workspaces.", + "memberCount": 2, + "isBlacklisted": False, + "publisherid": "299401", + "isPrivateNetworkEntity": False, + "isDomainNonTrivial": True, + "privateNetworkMeta": "", + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", + }, + "highlight": {"summary": "BLS BBOT api test."}, + } + ], + "meta": { + "queryText": "blacklanternsecurity", + "total": { + "collection": 0, + "request": 0, + "workspace": 1, + "api": 0, + "team": 0, + "user": 0, + "flow": 0, + "apiDefinition": 0, + "privateNetworkFolder": 0, + }, + "state": "AQ4", + "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "featureFlags": { + "enabledPublicResultCuration": True, + "boostByPopularity": True, + "reRankPostNormalization": True, + "enableUrlBarHostNameSearch": True, + }, + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}', + json={ + "meta": {"model": "workspace", "action": "find", "nextCursor": ""}, + "data": [ + { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "description": None, + "summary": "BLS public workspaces.", + "createdBy": "299401", + "updatedBy": "299401", + "team": None, + "createdAt": "2021-10-20T16:19:29", + "updatedAt": "2021-10-20T16:19:29", + "visibilityStatus": "public", + "profileInfo": { + "slug": "bbot-public", + "profileType": "team", + "profileId": "000000", + "publicHandle": "https://www.postman.com/blacklanternsecurity", + "publicImageURL": "", + "publicName": "BlackLanternSecurity", + "isVerified": False, + }, + } + ], + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + json={ + "workspace": { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "type": "personal", + "description": None, + "visibility": "public", + "createdBy": "00000000", + "updatedBy": "00000000", + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-17T08:57:16.000Z", + "collections": [ + { + "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + "name": "BBOT Public", + "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + }, + ], + "environments": [ + { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + } + ], + "apis": [], + } + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals", + json={ + "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "meta": {"model": "globals", "action": "find"}, + "data": { + "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "lastUpdatedBy": "00000000", + "lastRevision": 1637239113000, + "id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "values": [ + { + "key": "endpoint_url", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-18T12:38:33.000Z", + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + json={ + "environment": { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "owner": "00000000", + "createdAt": "2021-11-17T06:29:54.000Z", + "updatedAt": "2021-11-23T07:06:53.000Z", + "values": [ + { + "key": "temp_session_endpoint", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "isPublic": True, + } + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/collections/10197090-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + json={ + "collection": { + "info": { + "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0", + "name": "BBOT Public", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "updatedAt": "2021-11-17T07:13:16.000Z", + "createdAt": "2021-11-17T07:13:15.000Z", + "lastUpdatedBy": "00000000", + "uid": "00000000-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + }, + "item": [ + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": {"raw": "{{endpoint_url}}", "host": ["{{endpoint_url}}"]}, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + ], + } + }, + ) + + def check(self, module_test, events): + assert 1 == len( + [ + e + for e in events + if e.type == "CODE_REPOSITORY" + and "postman" in e.tags + and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public" + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity postman workspace" + assert 1 == len( + [ + e + for e in events + if e.type == "FILESYSTEM" + and "postman_workspaces/BlackLanternSecurity BBOT [Public]" in e.data["path"] + and "postman" in e.tags + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity postman workspace" From e19416d97556a5f3ee5a83979b2773b7bcc2ca07 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 18 Sep 2024 20:21:17 +0100 Subject: [PATCH 2/2] Add a description for the api_key --- bbot/modules/postman_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 807f2e9e9..be8b93aff 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -14,7 +14,7 @@ class postman_download(postman): "author": "@domwhewell-sage", } options = {"output_folder": "", "api_key": ""} - options_desc = {"output_folder": "Folder to download postman workspaces to"} + options_desc = {"output_folder": "Folder to download postman workspaces to", "api_key": "Postman API Key"} scope_distance_modifier = 2 async def setup(self):