merge dev

blacklanternsecurity · May 15, 2024 · 01bce76 · 01bce76
2 parents 72070a2 + 1f807f5
commit 01bce76
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 25 deletions.
diff --git a/bbot/core/flags.py b/bbot/core/flags.py
@@ -15,6 +15,7 @@
     "service-enum": "Identifies protocols running on open ports",
     "slow": "May take a long time to complete",
     "social-enum": "Enumerates social media",
+    "repo-enum": "Enumerates code repositories",
     "subdomain-enum": "Enumerates subdomains",
     "subdomain-hijack": "Detects hijackable subdomains",
     "web-basic": "Basic, non-intrusive web scan functionality",

diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
@@ -0,0 +1,48 @@
+import re
+from bbot.modules.base import BaseModule
+
+
+class code_repository(BaseModule):
+    watched_events = ["URL_UNVERIFIED"]
+    produced_events = ["CODE_REPOSITORY"]
+    meta = {"description": "Look for code repository links in webpages"}
+    flags = ["passive", "safe", "repo-enum"]
+
+    # platform name : (regex, case_sensitive)
+    code_repositories = {
+        "git": [
+            (r"github.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+            (r"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+        ],
+        "docker": (r"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+    }
+
+    scope_distance_modifier = 1
+
+    async def setup(self):
+        self.compiled_regexes = {}
+        for k, v in self.code_repositories.items():
+            if isinstance(v, list):
+                self.compiled_regexes[k] = [(re.compile(pattern), c) for pattern, c in v]
+            else:
+                pattern, c = v
+                self.compiled_regexes[k] = (re.compile(pattern), c)
+        return True
+
+    async def handle_event(self, event):
+        for platform, regexes in self.compiled_regexes.items():
+            if not isinstance(regexes, list):
+                regexes = [regexes]
+            for regex, case_sensitive in regexes:
+                for match in regex.finditer(event.data):
+                    url = match.group()
+                    if not case_sensitive:
+                        url = url.lower()
+                    repo_event = self.make_event(
+                        {"url": f"https://{url}"},
+                        "CODE_REPOSITORY",
+                        tags=platform,
+                        source=event,
+                    )
+                    repo_event.scope_distance = event.scope_distance
+                    await self.emit_event(repo_event)
diff --git a/bbot/test/test_step_2/module_tests/test_module_code_repository.py b/bbot/test/test_step_2/module_tests/test_module_code_repository.py
@@ -0,0 +1,59 @@
+from .base import ModuleTestBase
+
+
+class TestCodeRepository(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "excavate", "code_repository"]
+
+    async def setup_after_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/"}
+        respond_args = {
+            "response_data": """
+            <html>
+                <a href="https://github.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.org/blacklanternsecurity/bbot"/>
+                <a href="https://hub.docker.com/r/blacklanternsecurity/bbot"/>
+            </html>
+            """
+        }
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        assert 4 == len([e for e in events if e.type == "CODE_REPOSITORY"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://github.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "docker" in e.tags
+                and e.data["url"] == "https://hub.docker.com/r/blacklanternsecurity/bbot"
+            ]
+        )
diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md
@@ -58,6 +58,7 @@
 | censys               | scan     | Yes             | Query the Censys API                                                                                                                    | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | certspotter          | scan     | No              | Query Certspotter's API for subdomains                                                                                                  | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | chaos                | scan     | Yes             | Query ProjectDiscovery's Chaos API for subdomains                                                                                       | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
+| code_repository      | scan     | No              | Look for code repository links in webpages                                                                                              | passive, repo-enum, safe                                                       | URL_UNVERIFIED                                                                                                                          | CODE_REPOSITORY                                             |
 | columbus             | scan     | No              | Query the Columbus Project API for subdomains                                                                                           | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | credshed             | scan     | Yes             | Send queries to your own credshed server to check for known credentials of your targets                                                 | passive, safe                                                                  | DNS_NAME                                                                                                                                | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME          |
 | crobat               | scan     | No              | Query Project Crobat for subdomains                                                                                                     | passive, safe                                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |