Merge branch 'dev' into apkpure

bbot/core/event/base.py

@@ -1537,6 +1537,13 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     _always_emit_tags = ["target"]
 
 
+class MOBILE_APP(DictEvent):
+    _always_emit = True
+
+    def _pretty_string(self):
+        return self.data["url"]
+
+
 def make_event(
     data,
     event_type=None,

bbot/modules/google_playstore.py

@@ -0,0 +1,93 @@
+from bbot.modules.base import BaseModule
+
+
+class google_playstore(BaseModule):
+    watched_events = ["ORG_STUB", "CODE_REPOSITORY"]
+    produced_events = ["MOBILE_APP"]
+    flags = ["passive", "safe", "code-enum"]
+    meta = {
+        "description": "Search for android applications on play.google.com",
+        "created_date": "2024-10-08",
+        "author": "@domwhewell-sage",
+    }
+
+    base_url = "https://play.google.com"
+
+    async def setup(self):
+        self.app_link_regex = self.helpers.re.compile(r"/store/apps/details\?id=([a-zA-Z0-9._-]+)")
+        return True
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "android" not in event.tags:
+                return False, "event is not an android repository"
+        return True
+
+    async def handle_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            await self.handle_url(event)
+        elif event.type == "ORG_STUB":
+            await self.handle_org_stub(event)
+
+    async def handle_url(self, event):
+        repo_url = event.data.get("url")
+        app_id = repo_url.split("id=")[1].split("&")[0]
+        await self.emit_event(
+            {"id": app_id, "url": repo_url},
+            "MOBILE_APP",
+            tags="android",
+            parent=event,
+            context=f'{{module}} extracted the mobile app name "{app_id}"  from: {repo_url}',
+        )
+
+    async def handle_org_stub(self, event):
+        org_name = event.data
+        self.verbose(f"Searching for any android applications for {org_name}")
+        for apk_name in await self.query(org_name):
+            valid_apk = await self.validate_apk(apk_name)
+            if valid_apk:
+                self.verbose(f"Got {apk_name} from playstore")
+                await self.emit_event(
+                    {"id": apk_name, "url": f"{self.base_url}/store/apps/details?id={apk_name}"},
+                    "MOBILE_APP",
+                    tags="android",
+                    parent=event,
+                    context=f'{{module}} searched play.google.com for apps belonging to "{org_name}" and found "{apk_name}" to be in scope',
+                )
+            else:
+                self.debug(f"Got {apk_name} from playstore app details does not contain any in-scope URLs or Emails")
+
+    async def query(self, query):
+        app_links = []
+        url = f"{self.base_url}/store/search?q={self.helpers.quote(query)}&c=apps"
+        r = await self.helpers.request(url)
+        if r is None:
+            return app_links
+        status_code = getattr(r, "status_code", 0)
+        try:
+            html_content = r.content.decode("utf-8")
+            # Use regex to find all app links
+            app_links = await self.helpers.re.findall(self.app_link_regex, html_content)
+        except Exception as e:
+            self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}")
+            return app_links
+        return app_links
+
+    async def validate_apk(self, apk_name):
+        """
+        Check the app details page the "App support" section will include URLs or Emails to the app developer
+        """
+        in_scope = False
+        url = f"{self.base_url}/store/apps/details?id={apk_name}"
+        r = await self.helpers.request(url)
+        if r is None:
+            return in_scope
+        status_code = getattr(r, "status_code", 0)
+        if status_code == 200:
+            html = r.text
+            in_scope_hosts = await self.scan.extract_in_scope_hostnames(html)
+            if in_scope_hosts:
+                in_scope = True
+        else:
+            self.warning(f"Failed to fetch {url} (HTTP status: {status_code})")
+        return in_scope

bbot/scanner/scanner.py

@@ -1063,7 +1063,7 @@ def dns_regexes_yara(self):
         Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules.
         """
         if self._dns_regexes_yara is None:
-            self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+")
+            self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)*")
         return self._dns_regexes_yara
 
     @property

bbot/test/test_step_1/test_regexes.py

@@ -376,18 +376,25 @@ async def test_regex_helper():
     # test yara hostname extractor helper
     scan = Scanner("evilcorp.com", "www.evilcorp.net", "evilcorp.co.uk")
     host_blob = """
+    https://evilcorp.com/
     https://asdf.evilcorp.com/
     https://asdf.www.evilcorp.net/
     https://asdf.www.evilcorp.co.uk/
     https://asdf.www.evilcorp.com/
     https://asdf.www.evilcorp.com/
+    https://test.api.www.evilcorp.net/
     """
     extracted = await scan.extract_in_scope_hostnames(host_blob)
     assert extracted == {
-        "asdf.www.evilcorp.net",
+        "evilcorp.co.uk",
+        "evilcorp.com",
+        "www.evilcorp.com",
         "asdf.evilcorp.com",
         "asdf.www.evilcorp.com",
-        "www.evilcorp.com",
+        "www.evilcorp.net",
+        "api.www.evilcorp.net",
+        "asdf.www.evilcorp.net",
+        "test.api.www.evilcorp.net",
         "asdf.www.evilcorp.co.uk",
         "www.evilcorp.co.uk",
     }

bbot/test/test_step_2/module_tests/test_module_google_playstore.py

@@ -0,0 +1,83 @@
+from .base import ModuleTestBase
+
+
+class TestGoogle_Playstore(ModuleTestBase):
+    modules_overrides = ["google_playstore", "speculate"]
+
+    async def setup_after_prep(self, module_test):
+        await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.99"]}})
+        module_test.httpx_mock.add_response(
+            url="https://play.google.com/store/search?q=blacklanternsecurity&c=apps",
+            text="""<!DOCTYPE html>
+            <html>
+            <head>
+            <title>"blacklanternsecurity" - Android Apps on Google Play</title>
+            </head>
+            <body>
+            <a href="/store/apps/details?id=com.bbot.test&pcampaignid=dontmatchme&pli=1"/>
+            <a href="/store/apps/details?id=com.bbot.other"/>
+            </body>
+            </html>""",
+        )
+        module_test.httpx_mock.add_response(
+            url="https://play.google.com/store/apps/details?id=com.bbot.test",
+            text="""<!DOCTYPE html>
+            <html>
+            <head>
+            <title>BBOT</title>
+            </head>
+            <body>
+            <meta name="appstore:developer_url" content="https://www.blacklanternsecurity.com">
+            </div>
+            </div>
+            </body>
+            </html>""",
+        )
+        module_test.httpx_mock.add_response(
+            url="https://play.google.com/store/apps/details?id=com.bbot.other",
+            text="""<!DOCTYPE html>
+            <html>
+            <head>
+            <title>BBOT</title>
+            </head>
+            <body>
+            <meta name="appstore:developer_url" content="">
+            <a href="mailto:[email protected]"></a>
+            </div>
+            </div>
+            </body>
+            </html>""",
+        )
+
+    def check(self, module_test, events):
+        assert len(events) == 6
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0
+            ]
+        ), "Failed to emit target DNS_NAME"
+        assert 1 == len(
+            [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0]
+        ), "Failed to find ORG_STUB"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "MOBILE_APP"
+                and "android" in e.tags
+                and e.data["id"] == "com.bbot.test"
+                and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.test"
+            ]
+        ), "Failed to find bbot android app"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "MOBILE_APP"
+                and "android" in e.tags
+                and e.data["id"] == "com.bbot.other"
+                and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.other"
+            ]
+        ), "Failed to find other bbot android app"