added ORG_STUB event type

bbot/core/event/base.py

@@ -393,7 +393,7 @@ def source(self, source):
         """
         if is_event(source):
             self._source = source
-            hosts_are_same = self.host == source.host
+            hosts_are_same = self.host and (self.host == source.host)
             if source.scope_distance >= 0:
                 new_scope_distance = int(source.scope_distance)
                 # only increment the scope distance if the host changes
@@ -752,6 +752,8 @@ class ASN(DictEvent):
 
 
 class CODE_REPOSITORY(DictHostEvent):
+    _always_emit = True
+
     class _data_validator(BaseModel):
         url: str
         _validate_url = field_validator("url")(validators.validate_url)

bbot/core/helpers/helper.py

@@ -157,8 +157,12 @@ def __getattribute__(self, attr):
                         # then try web
                         return getattr(self.web, attr)
                     except AttributeError:
-                        # then die
-                        raise AttributeError(f'Helper has no attribute "{attr}"')
+                        try:
+                            # then try validators
+                            return getattr(self.validators, attr)
+                        except AttributeError:
+                            # then die
+                            raise AttributeError(f'Helper has no attribute "{attr}"')
 
 
 class DummyModule(BaseModule):

bbot/core/helpers/misc.py

@@ -504,12 +504,13 @@ def is_port(p):
     return p and p.isdigit() and 0 <= int(p) <= 65535
 
 
-def is_dns_name(d):
+def is_dns_name(d, include_local=True):
     """
     Determines if the given string is a valid DNS name.
 
     Args:
         d (str): The string to be checked.
+        include_local (bool): Consider local hostnames to be valid (hostnames without periods)
 
     Returns:
         bool: True if the string is a valid DNS name, False otherwise.
@@ -519,14 +520,17 @@ def is_dns_name(d):
         True
         >>> is_dns_name('localhost')
         True
+        >>> is_dns_name('localhost', include_local=False)
+        False
         >>> is_dns_name('192.168.1.1')
         False
     """
     if is_ip(d):
         return False
     d = smart_decode(d)
-    if bbot_regexes.hostname_regex.match(d):
-        return True
+    if include_local:
+        if bbot_regexes.hostname_regex.match(d):
+            return True
     if bbot_regexes.dns_name_regex.match(d):
         return True
     return False

bbot/core/helpers/validators.py

@@ -280,3 +280,11 @@ def soft_validate(s, t):
         return True
     except ValueError:
         return False
+
+
+def is_email(email):
+    try:
+        validate_email(email)
+        return True
+    except ValueError:
+        return False

bbot/modules/github_codesearch.py

@@ -6,8 +6,14 @@ class github_codesearch(github):
     produced_events = ["CODE_REPOSITORY", "URL_UNVERIFIED"]
     flags = ["passive", "subdomain-enum", "safe"]
     meta = {"description": "Query Github's API for code containing the target domain name", "auth_required": True}
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Github token"}
+    options = {"api_key": "", "limit": 100}
+    options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"}
+
+    github_raw_url = "https://raw.githubusercontent.com/"
+
+    async def setup(self):
+        self.limit = self.config.get("limit", 100)
+        return await super().setup()
 
     async def handle_event(self, event):
         query = self.make_query(event)
@@ -27,6 +33,7 @@ async def query(self, query):
         repos = {}
         url = f"{self.base_url}/search/code?per_page=100&type=Code&q={self.helpers.quote(query)}&page=" + "{page}"
         agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        num_results = 0
         try:
             async for r in agen:
                 if r is None:
@@ -35,6 +42,8 @@ async def query(self, query):
                 if status_code == 429:
                     "Github is rate-limiting us (HTTP status: 429)"
                     break
+                if status_code != 200:
+                    break
                 try:
                     j = r.json()
                 except Exception as e:
@@ -52,10 +61,14 @@ async def query(self, query):
                             repos[repo_url].append(raw_url)
                         except KeyError:
                             repos[repo_url] = [raw_url]
+                        num_results += 1
+                        if num_results >= self.limit:
+                            break
+                if num_results >= self.limit:
+                    break
         finally:
             agen.aclose()
         return repos
 
-    @staticmethod
-    def raw_url(url):
-        return url.replace("https://github.com/", "https://raw.githubusercontent.com/").replace("/blob/", "/")
+    def raw_url(self, url):
+        return url.replace("https://github.com/", self.github_raw_url).replace("/blob/", "/")

bbot/modules/github_org.py

@@ -2,31 +2,85 @@
 
 
 class github_org(github):
-    watched_events = ["DNS_NAME"]
+    watched_events = ["ORG_STUB", "SOCIAL"]
     produced_events = ["CODE_REPOSITORY"]
     flags = ["passive", "subdomain-enum", "safe"]
-    meta = {"description": "Query Github's API for a organization and member repositories"}
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Github token"}
+    meta = {"description": "Query Github's API for organization and member repositories"}
+    options = {"api_key": "", "include_members": True, "include_member_repos": False}
+    options_desc = {
+        "api_key": "Github token",
+        "include_members": "Enumerate organization members",
+        "include_member_repos": "Also enumerate organization members' repositories",
+    }
+
+    scope_distance_modifier = 2
+
+    async def setup(self):
+        self.include_members = self.config.get("include_members", True)
+        self.include_member_repos = self.config.get("include_member_repos", False)
+        return await super().setup()
+
+    async def filter_event(self, event):
+        if event.type == "SOCIAL":
+            if event.data.get("platform", "") != "github":
+                return False, "event is not a github profile"
+            # reject org members if the setting isn't enabled
+            # this prevents gathering of org member repos
+            if (not self.include_member_repos) and ("github-org-member" in event.tags):
+                return False, "include_member_repos is False"
+        return True
 
     async def handle_event(self, event):
-        domain = self.make_query(event)
-        potential_org = domain.split(".")[0]
-        if await self.validate_org(potential_org, domain):
-            self.verbose(f"Search for any repositorys belonging to {potential_org} and its members")
-            for repo_url in await self.query(potential_org):
-                self.emit_event({"url": repo_url}, "CODE_REPOSITORY", source=event)
-        else:
-            self.warning(f"Unable to validate {potential_org} is within the scope of this assesment, skipping...")
+        # handle github profile
+        if event.type == "SOCIAL":
+            user = event.data.get("profile_name", "")
+            in_scope = False
+            if "github-org-member" in event.tags:
+                is_org = False
+            elif "github-org" in event.tags:
+                is_org = True
+                in_scope = True
+            else:
+                is_org, in_scope = await self.validate_org(user)
 
-    async def query(self, query):
-        repos = []
-        org_repos = await self.query_org_repos(query)
-        repos.extend(org_repos)
-        for member in await self.query_org_members(query):
-            member_repos = await self.query_user_repos(member)
-            repos.extend(member_repos)
-        return repos
+            # find repos from user/org (SOCIAL --> CODE_REPOSITORY)
+            repos = []
+            if is_org:
+                if in_scope:
+                    self.verbose(f"Searching for repos belonging to organization {user}")
+                    repos = await self.query_org_repos(user)
+                else:
+                    self.verbose(f"Organization {user} does not appear to be in-scope")
+            elif "github-org-member" in event.tags:
+                self.verbose(f"Searching for repos belonging to user {user}")
+                repos = await self.query_user_repos(user)
+            for repo_url in repos:
+                repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", source=event)
+                repo_event.scope_distance = event.scope_distance
+                self.emit_event(repo_event)
+
+            # find members from org (SOCIAL --> SOCIAL)
+            if is_org and self.include_members:
+                self.verbose(f"Searching for any members belonging to {user}")
+                org_members = await self.query_org_members(user)
+                for member in org_members:
+                    event_data = {"platform": "github", "profile_name": member, "url": f"https://github.com/{member}"}
+                    member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", source=event)
+                    self.emit_event(member_event)
+
+        # find valid orgs from stub (ORG_STUB --> SOCIAL)
+        elif event.type == "ORG_STUB":
+            user = event.data
+            self.verbose(f"Validating whether the organization {user} is within our scope...")
+            is_org, in_scope = await self.validate_org(user)
+            if not is_org or not in_scope:
+                self.verbose(f"Unable to validate that {user} is in-scope, skipping...")
+                return
+
+            event_data = {"platform": "github", "profile_name": user, "url": f"https://github.com/{user}"}
+            github_org_event = self.make_event(event_data, "SOCIAL", tags="github-org", source=event)
+            github_org_event.scope_distance = event.scope_distance
+            self.emit_event(github_org_event)
 
     async def query_org_repos(self, query):
         repos = []
@@ -40,6 +94,8 @@ async def query_org_repos(self, query):
                 if status_code == 403:
                     self.warning("Github is rate-limiting us (HTTP status: 403)")
                     break
+                if status_code != 200:
+                    break
                 try:
                     j = r.json()
                 except Exception as e:
@@ -66,6 +122,8 @@ async def query_org_members(self, query):
                 if status_code == 403:
                     self.warning("Github is rate-limiting us (HTTP status: 403)")
                     break
+                if status_code != 200:
+                    break
                 try:
                     j = r.json()
                 except Exception as e:
@@ -92,6 +150,8 @@ async def query_user_repos(self, query):
                 if status_code == 403:
                     self.warning("Github is rate-limiting us (HTTP status: 403)")
                     break
+                if status_code != 200:
+                    break
                 try:
                     j = r.json()
                 except Exception as e:
@@ -106,23 +166,35 @@ async def query_user_repos(self, query):
             agen.aclose()
         return repos
 
-    async def validate_org(self, input, domain):
-        self.verbose(f"Validating the organization {input} is within our scope...")
+    async def validate_org(self, org):
+        is_org = False
         in_scope = False
-        url = f"{self.base_url}/orgs/{input}"
+        url = f"{self.base_url}/orgs/{org}"
         r = await self.helpers.request(url, headers=self.headers)
         if r is None:
-            return in_scope
+            return is_org, in_scope
         status_code = getattr(r, "status_code", 0)
         if status_code == 403:
             self.warning("Github is rate-limiting us (HTTP status: 403)")
+            return is_org, in_scope
+        if status_code == 200:
+            is_org = True
         try:
             json = r.json()
         except Exception as e:
             self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return in_scope
-        blog = json.get("blog", "")
-        if domain in blog:
-            self.verbose(f"{input} is within the scope of this assesment")
-            in_scope = True
-        return in_scope
+            return is_org, in_scope
+        for k, v in json.items():
+            if (
+                isinstance(v, str)
+                and (
+                    self.helpers.is_dns_name(v, include_local=False)
+                    or self.helpers.is_url(v)
+                    or self.helpers.is_email(v)
+                )
+                and self.scan.in_scope(v)
+            ):
+                self.verbose(f'Found in-scope key "{k}": "{v}" for {org}, it appears to be in-scope')
+                in_scope = True
+                break
+        return is_org, in_scope

bbot/modules/internal/speculate.py

@@ -19,8 +19,10 @@ class speculate(BaseInternalModule):
         "IP_ADDRESS",
         "HTTP_RESPONSE",
         "STORAGE_BUCKET",
+        "SOCIAL",
+        "AZURE_TENANT",
     ]
-    produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING"]
+    produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"]
     flags = ["passive"]
     meta = {"description": "Derive certain event types from others by common sense"}
 
@@ -37,6 +39,7 @@ async def setup(self):
         self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()])
         self.range_to_ip = True
         self.dns_resolution = self.scan.config.get("dns_resolution", True)
+        self.org_stubs_seen = set()
 
         port_string = self.config.get("ports", "80,443")
 
@@ -120,6 +123,28 @@ async def handle_event(self, event):
         # storage buckets etc.
         self.helpers.cloud.speculate(event)
 
+        # ORG_STUB from TLD, SOCIAL, AZURE_TENANT
+        org_stubs = set()
+        if event.type == "DNS_NAME" and event.scope_distance == 0:
+            tld_stub = getattr(self.helpers.tldextract(event.data), "domain", "")
+            if tld_stub:
+                org_stubs.add(tld_stub)
+        elif event.type == "SOCIAL":
+            stub = event.data.get("stub", "")
+            if stub:
+                org_stubs.add(stub.lower())
+        elif event.type == "AZURE_TENANT":
+            tenant_names = event.data.get("tenant-names", [])
+            org_stubs.update(set(tenant_names))
+        for stub in org_stubs:
+            stub_hash = hash(stub)
+            if stub_hash not in self.org_stubs_seen:
+                self.org_stubs_seen.add(stub_hash)
+                stub_event = self.make_event(stub, "ORG_STUB", source=event)
+                if event.scope_distance > 0:
+                    stub_event.scope_distance = event.scope_distance
+                self.emit_event(stub_event)
+
     async def filter_event(self, event):
         # don't accept IP_RANGE --> IP_ADDRESS events from self
         if str(event.module) == "speculate":

bbot/modules/secretsdb.py

@@ -18,6 +18,8 @@ class secretsdb(BaseModule):
         "signatures": "File path or URL to YAML signatures",
     }
     deps_pip = ["pyyaml~=6.0"]
+    # accept any HTTP_RESPONSE including out-of-scope ones (such as from github_codesearch)
+    scope_distance_modifier = 3
 
     async def setup(self):
         self.rules = []