Merge pull request #1644 from domwhewell-sage/deleted_forks

Trufflehog enhancement scan forks

bbot/modules/trufflehog.py

@@ -3,7 +3,7 @@
 
 
 class trufflehog(BaseModule):
-    watched_events = ["FILESYSTEM"]
+    watched_events = ["CODE_REPOSITORY", "FILESYSTEM"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["passive", "safe", "code-enum"]
     meta = {
@@ -13,14 +13,16 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.75.1",
+        "version": "3.81.7",
         "only_verified": True,
         "concurrency": 8,
+        "deleted_forks": False,
     }
     options_desc = {
         "version": "trufflehog version",
         "only_verified": "Only report credentials that have been verified",
         "concurrency": "Number of concurrent workers",
+        "deleted_forks": "Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.",
     }
     deps_ansible = [
         {
@@ -39,25 +41,63 @@ class trufflehog(BaseModule):
     async def setup(self):
         self.verified = self.config.get("only_verified", True)
         self.concurrency = int(self.config.get("concurrency", 8))
+
+        self.deleted_forks = self.config.get("deleted_forks", False)
+        self.github_token = ""
+        if self.deleted_forks:
+            self.warning(
+                f"Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
+            )
+            for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
+                module_config = self.scan.config.get("modules", {}).get(module_name, {})
+                api_key = module_config.get("api_key", "")
+                if api_key:
+                    self.github_token = api_key
+                    break
+
+            # soft-fail if we don't have a github token as well
+            if not self.github_token:
+                self.deleted_forks = False
+                return None, "A github api_key must be provided to the github modules for deleted forks to be scanned"
+        return True
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if self.deleted_forks:
+                if "git" not in event.tags:
+                    return False, "Module only accepts git CODE_REPOSITORY events"
+                if "github" not in event.data["url"]:
+                    return False, "Module only accepts github CODE_REPOSITORY events"
+            else:
+                return False, "Deleted forks is not enabled"
         return True
 
     async def handle_event(self, event):
-        path = event.data["path"]
         description = event.data.get("description", "")
-        if "git" in event.tags:
-            module = "git"
-        elif "docker" in event.tags:
-            module = "docker"
+        if event.type == "CODE_REPOSITORY":
+            path = event.data["url"]
+            if "git" in event.tags:
+                module = "github-experimental"
+        else:
+            path = event.data["path"]
+            if "git" in event.tags:
+                module = "git"
+            elif "docker" in event.tags:
+                module = "docker"
+            else:
+                module = "filesystem"
+        if event.type == "CODE_REPOSITORY":
+            host = event.host
         else:
-            module = "filesystem"
+            host = str(event.parent.host)
         async for decoder_name, detector_name, raw_result, verified, source_metadata in self.execute_trufflehog(
             module, path
         ):
             if verified:
                 data = {
                     "severity": "High",
                     "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
-                    "host": str(event.parent.host),
+                    "host": host,
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
@@ -70,7 +110,7 @@ async def handle_event(self, event):
             else:
                 data = {
                     "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
-                    "host": str(event.parent.host),
+                    "host": host,
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
@@ -99,6 +139,12 @@ async def execute_trufflehog(self, module, path):
         elif module == "filesystem":
             command.append("filesystem")
             command.append(path)
+        elif module == "github-experimental":
+            command.append("github-experimental")
+            command.append("--repo=" + path)
+            command.append("--object-discovery")
+            command.append("--delete-cached-data")
+            command.append("--token=" + self.github_token)
 
         stats_file = self.helpers.tempfile_tail(callback=self.log_trufflehog_status)
         try: