From 4eb472e36db9835dc33ab527befa00f342a90526 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 1 Oct 2024 10:34:30 +0100 Subject: [PATCH] Trufflehog now accepts postman workspaces --- bbot/modules/trufflehog.py | 5 + .../module_tests/test_module_trufflehog.py | 326 +++++++++++++++++- 2 files changed, 320 insertions(+), 11 deletions(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 61aa2306d..db3497ac9 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -99,6 +99,8 @@ async def handle_event(self, event): module = "git" elif "docker" in event.tags: module = "docker" + elif "postman" in event.tags: + module = "postman" else: module = "filesystem" if event.type == "CODE_REPOSITORY": @@ -164,6 +166,9 @@ async def execute_trufflehog(self, module, path): elif module == "docker": command.append("docker") command.append("--image=file://" + path) + elif module == "postman": + command.append("postman") + command.append("--workspace-paths=" + path) elif module == "filesystem": command.append("filesystem") command.append(path) diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py index 68285a001..a838ad6ab 100644 --- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py +++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py @@ -9,6 +9,7 @@ class TestTrufflehog(ModuleTestBase): + config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}} modules_overrides = [ "github_org", "speculate", @@ -17,6 +18,8 @@ class TestTrufflehog(ModuleTestBase): "github_workflows", "dockerhub", "docker_pull", + "postman", + "postman_download", "trufflehog", ] @@ -24,6 +27,37 @@ class TestTrufflehog(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response(url="https://api.github.com/zen") + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/me", + json={ + "user": { + "id": 000000, + "username": "test_key", + "email": "blacklanternsecurity@test.com", + "fullName": "Test Key", + "avatar": "", + "isPublic": True, + "teamId": 0, + "teamDomain": "", + "roles": ["user"], + }, + "operations": [ + {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0}, + {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0}, + {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0}, + {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0}, + {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0}, + {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0}, + {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0}, + {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0}, + ], + }, + ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity", json={ @@ -813,6 +847,248 @@ async def setup_before_prep(self, module_test): ) async def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}', + json={ + "data": [ + { + "score": 611.41156, + "normalizedScore": 23, + "document": { + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "blacklanternsecurity", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", + "slug": "bbot-public", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "blacklanternsecurity", + "name": "BlackLanternSecurity BBOT [Public]", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", + "privateNetworkFolders": [], + "isPublisherVerified": False, + "publisherType": "team", + "curatedInList": [], + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "", + "popularity": 5, + "isPublic": True, + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "BLS public workspaces.", + "memberCount": 2, + "isBlacklisted": False, + "publisherid": "299401", + "isPrivateNetworkEntity": False, + "isDomainNonTrivial": True, + "privateNetworkMeta": "", + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", + }, + "highlight": {"summary": "BLS BBOT api test."}, + }, + ], + "meta": { + "queryText": "blacklanternsecurity", + "total": { + "collection": 0, + "request": 0, + "workspace": 1, + "api": 0, + "team": 0, + "user": 0, + "flow": 0, + "apiDefinition": 0, + "privateNetworkFolder": 0, + }, + "state": "AQ4", + "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "featureFlags": { + "enabledPublicResultCuration": True, + "boostByPopularity": True, + "reRankPostNormalization": True, + "enableUrlBarHostNameSearch": True, + }, + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}', + json={ + "meta": {"model": "workspace", "action": "find", "nextCursor": ""}, + "data": [ + { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "description": None, + "summary": "BLS public workspaces.", + "createdBy": "299401", + "updatedBy": "299401", + "team": None, + "createdAt": "2021-10-20T16:19:29", + "updatedAt": "2021-10-20T16:19:29", + "visibilityStatus": "public", + "profileInfo": { + "slug": "bbot-public", + "profileType": "team", + "profileId": "000000", + "publicHandle": "https://www.postman.com/blacklanternsecurity", + "publicImageURL": "", + "publicName": "BlackLanternSecurity", + "isVerified": False, + }, + } + ], + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + json={ + "workspace": { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "type": "personal", + "description": None, + "visibility": "public", + "createdBy": "00000000", + "updatedBy": "00000000", + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-17T08:57:16.000Z", + "collections": [ + { + "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + "name": "BBOT Public", + "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + }, + ], + "environments": [ + { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + } + ], + "apis": [], + } + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals", + json={ + "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "meta": {"model": "globals", "action": "find"}, + "data": { + "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "lastUpdatedBy": "00000000", + "lastRevision": 1637239113000, + "id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "values": [ + { + "key": "endpoint_url", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-18T12:38:33.000Z", + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + json={ + "environment": { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "owner": "00000000", + "createdAt": "2021-11-17T06:29:54.000Z", + "updatedAt": "2021-11-23T07:06:53.000Z", + "values": [ + { + "key": "temp_session_endpoint", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "isPublic": True, + } + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + json={ + "collection": { + "info": { + "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0", + "name": "BBOT Public", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "updatedAt": "2021-11-17T07:13:16.000Z", + "createdAt": "2021-11-17T07:13:15.000Z", + "lastUpdatedBy": "00000000", + "uid": "172983-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + }, + "item": [ + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": { + "raw": "https://admin:admin@the-internet.herokuapp.com/basic_auth", + "host": ["https://admin:admin@the-internet.herokuapp.com/basic_auth"], + }, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": { + "raw": "https://admin:admin@internal.host.com", + "host": ["https://admin:admin@internal.host.com"], + }, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + ], + } + }, + ) temp_path = Path("/tmp/.bbot_test") temp_repo_path = temp_path / "test_keys" shutil.rmtree(temp_repo_path, ignore_errors=True) @@ -850,21 +1126,25 @@ def check(self, module_test, events): e for e in events if e.type == "VULNERABILITY" - and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com") + and ( + e.data["host"] == "hub.docker.com" + or e.data["host"] == "github.com" + or e.data["host"] == "www.postman.com" + ) and "Verified Secret Found." in e.data["description"] and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"] and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"] ] - # Trufflehog should find 3 verifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder - assert 3 == len(vuln_events), "Failed to find secret in events" + # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + assert 4 == len(vuln_events), "Failed to find secret in events" github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) assert folder.is_dir(), "Destination folder doesn't exist" with open(folder / "keys.txt") as f: content = f.read() assert content == self.file_content, "File content doesn't match" - filesystem_events = [e.parent for e in vuln_events if "bbot" in e.data["description"]] - assert len(filesystem_events) == 3 + filesystem_events = [e.parent for e in vuln_events] + assert len(filesystem_events) == 4 assert all([e.type == "FILESYSTEM" for e in filesystem_events]) assert 1 == len( [ @@ -889,30 +1169,44 @@ def check(self, module_test, events): and Path(e.data["path"]).is_file() ] ), "Docker image file does not exist" + assert 1 == len( + [ + e + for e in filesystem_events + if e.data["path"].endswith( + "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip" + ) + and Path(e.data["path"]).is_file() + ] + ), "Failed to find blacklanternsecurity postman workspace" class TestTrufflehog_NonVerified(TestTrufflehog): - config_overrides = {"modules": {"trufflehog": {"only_verified": False}}} + config_overrides = {"modules": {"trufflehog": {"only_verified": False}, "postman_download": {"api_key": "asdf"}}} def check(self, module_test, events): finding_events = [ e for e in events if e.type == e.type == "FINDING" - and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com") + and ( + e.data["host"] == "hub.docker.com" + or e.data["host"] == "github.com" + or e.data["host"] == "www.postman.com" + ) and "Potential Secret Found." in e.data["description"] and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"] ] - # Trufflehog should find 3 unverifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder - assert 3 == len(finding_events), "Failed to find secret in events" + # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + assert 4 == len(finding_events), "Failed to find secret in events" github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) assert folder.is_dir(), "Destination folder doesn't exist" with open(folder / "keys.txt") as f: content = f.read() assert content == self.file_content, "File content doesn't match" - filesystem_events = [e.parent for e in finding_events if "bbot" in e.data["description"]] - assert len(filesystem_events) == 3 + filesystem_events = [e.parent for e in finding_events] + assert len(filesystem_events) == 4 assert all([e.type == "FILESYSTEM" for e in filesystem_events]) assert 1 == len( [ @@ -937,3 +1231,13 @@ def check(self, module_test, events): and Path(e.data["path"]).is_file() ] ), "Docker image file does not exist" + assert 1 == len( + [ + e + for e in filesystem_events + if e.data["path"].endswith( + "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip" + ) + and Path(e.data["path"]).is_file() + ] + ), "Failed to find blacklanternsecurity postman workspace"