From f0eca9a597f84952d986c6207dca9c4516e74e18 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Thu, 7 Nov 2024 18:53:22 +0000 Subject: [PATCH 01/12] Added new module jadx --- bbot/modules/jadx.py | 111 ++++++++++++++++++ .../module_tests/test_module_jadx.py | 50 ++++++++ 2 files changed, 161 insertions(+) create mode 100644 bbot/modules/jadx.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_jadx.py diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py new file mode 100644 index 000000000..282547d5f --- /dev/null +++ b/bbot/modules/jadx.py @@ -0,0 +1,111 @@ +from pathlib import Path +from subprocess import CalledProcessError +from bbot.modules.internal.base import BaseModule + + +class jadx(BaseModule): + watched_events = ["FILESYSTEM"] + produced_events = ["FILESYSTEM"] + flags = ["passive"] + meta = { + "description": "Decompile APKs and XAPKs using JADX", + "created_date": "2024-11-04", + "author": "@domwhewell-sage", + } + options = { + "threads": 4, + } + options_desc = { + "threads": "Maximum jadx threads for extracting apk's, default: 4", + } + deps_ansible = [ + { + "name": "Install latest JRE (Debian)", + "package": {"name": ["default-jre"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian'", + }, + { + "name": "Install latest JRE (Arch)", + "package": {"name": ["jre-openjdk"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Archlinux'", + }, + { + "name": "Install latest JRE (Fedora)", + "package": {"name": ["java-openjdk-headless"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'RedHat'", + }, + { + "name": "Install latest JRE (Alpine)", + "package": {"name": ["openjdk11"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Alpine'", + }, + { + "name": "Create jadx directory", + "file": {"path": "#{BBOT_TOOLS}/jadx", "state": "directory", "mode": "0755"}, + }, + { + "name": "Download jadx", + "unarchive": { + "src": "https://github.com/skylot/jadx/releases/download/v1.5.0/jadx-1.5.0.zip", + "include": ["lib/jadx-1.5.0-all.jar", "bin/jadx"], + "dest": "#{BBOT_TOOLS}/jadx", + "remote_src": True, + }, + }, + ] + + allowed_file_types = ["java archive", "android application package"] + + async def setup(self): + self.threads = self.config.get("threads", 4) + return True + + async def filter_event(self, event): + if "file" in event.tags: + if not event.data["magic_description"].lower() in self.allowed_file_types: + return False, f"Jadx is not able to decompile this file type: {event.data['magic_description']}" + else: + return False, "Event is not a file" + return True + + async def handle_event(self, event): + path = Path(event.data["path"]) + output_dir = path.parent / path.name.replace(".", "_") + self.helpers.mkdir(output_dir) + success = await self.decompile_apk(path, output_dir) + + # If jadx was able to decompile the java archive, emit an event + if success: + await self.emit_event( + {"path": str(output_dir)}, + "FILESYSTEM", + tags="folder", + parent=event, + context=f'extracted "{path}" to: {output_dir}', + ) + else: + output_dir.rmdir() + + async def decompile_apk(self, path, output_dir): + command = [ + f"{self.scan.helpers.tools_dir}/jadx/bin/jadx", + "--threads-count", + self.threads, + "--output-dir", + str(output_dir), + str(path), + ] + try: + output = await self.run_process(command, check=True) + except CalledProcessError as e: + self.warning(f"Error decompiling {path}. STDERR: {repr(e.stderr)}") + return False + if not Path(output_dir / "resources").exists() and not Path(output_dir / "sources").exists(): + self.warning(f"JADX was unable to decompile {path}.") + self.warning(output) + return False + return True diff --git a/bbot/test/test_step_2/module_tests/test_module_jadx.py b/bbot/test/test_step_2/module_tests/test_module_jadx.py new file mode 100644 index 000000000..f949580d2 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_jadx.py @@ -0,0 +1,50 @@ +from pathlib import Path +from .base import ModuleTestBase, tempapkfile + + +class TestJadx(ModuleTestBase): + modules_overrides = ["apkpure", "google_playstore", "speculate", "jadx"] + apk_file = tempapkfile() + + async def setup_after_prep(self, module_test): + await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.99"]}}) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/search?q=blacklanternsecurity&c=apps", + text=""" + + + "blacklanternsecurity" - Android Apps on Google Play + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/apps/details?id=com.bbot.test", + text=""" + + + BBOT + + + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest", + content=self.apk_file, + ) + + def check(self, module_test, events): + extract_event = [ + e + for e in events + if e.type == "FILESYSTEM" and "com_bbot_test_xapk" in e.data["path"] and "folder" in e.tags + ] + assert 1 == len(extract_event), "Failed to extract apk" + extract_path = Path(extract_event[0].data["path"]) + assert extract_path.is_dir(), "Destination apk doesn't exist" From 4f3b5186d6c937a8b49cc925d6e45e7749374a36 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Thu, 7 Nov 2024 19:08:24 +0000 Subject: [PATCH 02/12] Add safe flag --- bbot/modules/jadx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 282547d5f..5a773c8f8 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -6,7 +6,7 @@ class jadx(BaseModule): watched_events = ["FILESYSTEM"] produced_events = ["FILESYSTEM"] - flags = ["passive"] + flags = ["passive", "safe"] meta = { "description": "Decompile APKs and XAPKs using JADX", "created_date": "2024-11-04", From 3e67a9f05a552edf2ed20cd4f0c72d92a09055f0 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 9 Nov 2024 09:53:12 +0000 Subject: [PATCH 03/12] Added test to check the detected file type --- bbot/modules/jadx.py | 2 +- .../module_tests/test_module_jadx.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 5a773c8f8..829542f99 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -67,7 +67,7 @@ async def setup(self): async def filter_event(self, event): if "file" in event.tags: if not event.data["magic_description"].lower() in self.allowed_file_types: - return False, f"Jadx is not able to decompile this file type: {event.data['magic_description']}" + return False, f"Jadx is not able to decompile this file type" else: return False, "Event is not a file" return True diff --git a/bbot/test/test_step_2/module_tests/test_module_jadx.py b/bbot/test/test_step_2/module_tests/test_module_jadx.py index f949580d2..9eb3c1537 100644 --- a/bbot/test/test_step_2/module_tests/test_module_jadx.py +++ b/bbot/test/test_step_2/module_tests/test_module_jadx.py @@ -1,5 +1,6 @@ from pathlib import Path -from .base import ModuleTestBase, tempapkfile +from bbot.core.helpers.libmagic import get_magic_info +from bbot.test.test_step_2.module_tests.base import ModuleTestBase, tempapkfile class TestJadx(ModuleTestBase): @@ -37,14 +38,18 @@ async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( url="https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest", content=self.apk_file, + headers={ + "Content-Type": "application/vnd.android.package-archive", + "Content-Disposition": "attachment; filename=com.bbot.test.xapk", + }, ) def check(self, module_test, events): - extract_event = [ - e - for e in events - if e.type == "FILESYSTEM" and "com_bbot_test_xapk" in e.data["path"] and "folder" in e.tags - ] + filesystem_events = [e for e in events if e.type == "FILESYSTEM"] + apk_event = [e for e in filesystem_events if "file" in e.tags] + extension, mime_type, description, confidence = get_magic_info(apk_event[0].data["path"]) + assert description == "Android application package", f"Downloaded file was detected as {description}" + extract_event = [e for e in filesystem_events if "folder" in e.tags] assert 1 == len(extract_event), "Failed to extract apk" extract_path = Path(extract_event[0].data["path"]) assert extract_path.is_dir(), "Destination apk doesn't exist" From 455051d9bf3855f75b043404bbdaf9d14da1de19 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 9 Nov 2024 15:05:49 +0000 Subject: [PATCH 04/12] Made changes to apkpure to create the file extension based on the Content-Disposition header --- bbot/modules/apkpure.py | 16 +++++++++++----- bbot/modules/jadx.py | 2 +- .../module_tests/test_module_apkpure.py | 10 ++++++---- .../test_step_2/module_tests/test_module_jadx.py | 4 ++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py index 210dfef0e..e7d22489c 100644 --- a/bbot/modules/apkpure.py +++ b/bbot/modules/apkpure.py @@ -45,9 +45,15 @@ async def download_apk(self, app_id): path = None url = f"https://d.apkpure.com/b/XAPK/{app_id}?version=latest" self.helpers.mkdir(self.output_dir / app_id) - file_destination = self.output_dir / app_id / f"{app_id}.xapk" - result = await self.helpers.download(url, warn=False, filename=file_destination) - if result: - self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}') - path = file_destination + response = await self.helpers.request(url, allow_redirects=True) + if response: + attachment = response.headers.get("Content-Disposition", "") + if "filename" in attachment: + extension = attachment.split('filename="')[-1].split(".")[-1].strip('"') + content = response.content + file_destination = self.output_dir / app_id / f"{app_id}.{extension}" + with open(file_destination, "wb") as f: + f.write(content) + self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}') + path = file_destination return path diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 829542f99..5a773c8f8 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -67,7 +67,7 @@ async def setup(self): async def filter_event(self, event): if "file" in event.tags: if not event.data["magic_description"].lower() in self.allowed_file_types: - return False, f"Jadx is not able to decompile this file type" + return False, f"Jadx is not able to decompile this file type: {event.data['magic_description']}" else: return False, "Event is not a file" return True diff --git a/bbot/test/test_step_2/module_tests/test_module_apkpure.py b/bbot/test/test_step_2/module_tests/test_module_apkpure.py index 17b8e0685..65919c62e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_apkpure.py +++ b/bbot/test/test_step_2/module_tests/test_module_apkpure.py @@ -37,6 +37,10 @@ async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( url="https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest", content=self.apk_file, + headers={ + "Content-Type": "application/vnd.android.package-archive", + "Content-Disposition": "attachment; filename=com.bbot.test.apk", + }, ) def check(self, module_test, events): @@ -61,9 +65,7 @@ def check(self, module_test, events): and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.test" ] ), "Failed to find bbot android app" - filesystem_event = [ - e for e in events if e.type == "FILESYSTEM" and "com.bbot.test.xapk" in e.data["path"] and "apk" in e.tags - ] + filesystem_event = [e for e in events if e.type == "FILESYSTEM" and "com.bbot.test.apk" in e.data["path"]] assert 1 == len(filesystem_event), "Failed to download apk" file = Path(filesystem_event[0].data["path"]) - assert file.is_file(), "Destination xapk doesn't exist" + assert file.is_file(), "Destination apk doesn't exist" diff --git a/bbot/test/test_step_2/module_tests/test_module_jadx.py b/bbot/test/test_step_2/module_tests/test_module_jadx.py index 9eb3c1537..f57dabad8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_jadx.py +++ b/bbot/test/test_step_2/module_tests/test_module_jadx.py @@ -40,7 +40,7 @@ async def setup_after_prep(self, module_test): content=self.apk_file, headers={ "Content-Type": "application/vnd.android.package-archive", - "Content-Disposition": "attachment; filename=com.bbot.test.xapk", + "Content-Disposition": "attachment; filename=com.bbot.test.apk", }, ) @@ -48,7 +48,7 @@ def check(self, module_test, events): filesystem_events = [e for e in events if e.type == "FILESYSTEM"] apk_event = [e for e in filesystem_events if "file" in e.tags] extension, mime_type, description, confidence = get_magic_info(apk_event[0].data["path"]) - assert description == "Android application package", f"Downloaded file was detected as {description}" + assert description == "Android Application Package", f"Downloaded file was detected as {description}" extract_event = [e for e in filesystem_events if "folder" in e.tags] assert 1 == len(extract_event), "Failed to extract apk" extract_path = Path(extract_event[0].data["path"]) From 64ddb453f49f6f135d58992870776e0f0ccaed4b Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 9 Nov 2024 17:06:59 +0000 Subject: [PATCH 05/12] Change fedora package --- bbot/modules/jadx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 5a773c8f8..8b873face 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -33,7 +33,7 @@ class jadx(BaseModule): }, { "name": "Install latest JRE (Fedora)", - "package": {"name": ["java-openjdk-headless"], "state": "present"}, + "package": {"name": ["java-latest-openjdk"], "state": "present"}, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, From 6aa2a5c7c9b8c50f1382ca0b7d3cf57d9972e2b5 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 9 Nov 2024 17:14:38 +0000 Subject: [PATCH 06/12] Change to regex to get the extension --- bbot/modules/apkpure.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py index e7d22489c..c17d25e2c 100644 --- a/bbot/modules/apkpure.py +++ b/bbot/modules/apkpure.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from bbot.modules.base import BaseModule @@ -49,11 +50,14 @@ async def download_apk(self, app_id): if response: attachment = response.headers.get("Content-Disposition", "") if "filename" in attachment: - extension = attachment.split('filename="')[-1].split(".")[-1].strip('"') - content = response.content - file_destination = self.output_dir / app_id / f"{app_id}.{extension}" - with open(file_destination, "wb") as f: - f.write(content) - self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}') - path = file_destination + match = re.search(r'filename="?([^"]+)"?', attachment) + if match: + filename = match.group(1) + extension = filename.split(".")[-1] + content = response.content + file_destination = self.output_dir / app_id / f"{app_id}.{extension}" + with open(file_destination, "wb") as f: + f.write(content) + self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}') + path = file_destination return path From 5a98cf326b9d8f4903f14030f91a248d77152580 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 9 Nov 2024 21:56:02 +0000 Subject: [PATCH 07/12] set the JAVA_HOME environment variable on fedora --- bbot/modules/jadx.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 8b873face..835d22343 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -37,6 +37,18 @@ class jadx(BaseModule): "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, + { + "name": "Set JAVA_HOME (Fedora)", + "command": "echo \"export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')\" >> ~/.bashrc", + "become": True, + "when": "ansible_facts['os_family'] == 'RedHat'", + }, + { + "name": "Use JAVA_HOME (Fedora)", + "command": "source ~/.bashrc", + "become": True, + "when": "ansible_facts['os_family'] == 'RedHat'", + }, { "name": "Install latest JRE (Alpine)", "package": {"name": ["openjdk11"], "state": "present"}, From 0173b873c799236b5ada407b20933c69f49a2f1a Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 10 Nov 2024 10:45:12 +0000 Subject: [PATCH 08/12] Please set the environment variable on fedora --- bbot/modules/jadx.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 835d22343..157c71b91 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -39,13 +39,7 @@ class jadx(BaseModule): }, { "name": "Set JAVA_HOME (Fedora)", - "command": "echo \"export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')\" >> ~/.bashrc", - "become": True, - "when": "ansible_facts['os_family'] == 'RedHat'", - }, - { - "name": "Use JAVA_HOME (Fedora)", - "command": "source ~/.bashrc", + "lineinfile": {"path": "/etc/profile.d/java.sh", "line": "export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')", "create": True}, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, From 4d69f3cdf21a108f8bd453db199e87132369a1b2 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 10 Nov 2024 10:46:53 +0000 Subject: [PATCH 09/12] lint --- bbot/modules/jadx.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 157c71b91..93553be63 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -33,13 +33,17 @@ class jadx(BaseModule): }, { "name": "Install latest JRE (Fedora)", - "package": {"name": ["java-latest-openjdk"], "state": "present"}, + "package": {"name": ["java-latest-openjdk-headless"], "state": "present"}, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, { "name": "Set JAVA_HOME (Fedora)", - "lineinfile": {"path": "/etc/profile.d/java.sh", "line": "export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')", "create": True}, + "lineinfile": { + "path": "/etc/profile.d/java.sh", + "line": "export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')", + "create": True, + }, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, From 41e8c29a9824640b0e355f2e897d643fbcf8fa87 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 10 Nov 2024 18:10:15 +0000 Subject: [PATCH 10/12] Set the java home in the jadx script --- bbot/modules/jadx.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 93553be63..45db9a81a 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -37,16 +37,6 @@ class jadx(BaseModule): "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, - { - "name": "Set JAVA_HOME (Fedora)", - "lineinfile": { - "path": "/etc/profile.d/java.sh", - "line": "export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')", - "create": True, - }, - "become": True, - "when": "ansible_facts['os_family'] == 'RedHat'", - }, { "name": "Install latest JRE (Alpine)", "package": {"name": ["openjdk11"], "state": "present"}, @@ -66,6 +56,16 @@ class jadx(BaseModule): "remote_src": True, }, }, + { + "name": "Set JAVA_HOME (Fedora)", + "lineinfile": { + "path": "#{BBOT_TOOLS}/jadx/bin/jadx", + "line": "JAVA_HOME=/usr", + "insertafter": "BOF", + }, + "become": True, + "when": "ansible_facts['os_family'] == 'RedHat'", + }, ] allowed_file_types = ["java archive", "android application package"] From be092b892a789e2664e88f5b3e80ade5ab18a0dc Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 11 Nov 2024 16:57:22 +0000 Subject: [PATCH 11/12] Ensure its inserted after the shebang --- bbot/modules/jadx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 45db9a81a..49f3450c9 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -61,7 +61,7 @@ class jadx(BaseModule): "lineinfile": { "path": "#{BBOT_TOOLS}/jadx/bin/jadx", "line": "JAVA_HOME=/usr", - "insertafter": "BOF", + "insertafter": "^#!", }, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", From c0be101cb841806a8fd0cbafd30e60d2904d3083 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 11 Nov 2024 18:02:32 +0000 Subject: [PATCH 12/12] Dont have to set JAVA_HOME can just install which to allow jadx startup script to find the java install --- bbot/modules/jadx.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 49f3450c9..86a8ecf89 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -33,7 +33,7 @@ class jadx(BaseModule): }, { "name": "Install latest JRE (Fedora)", - "package": {"name": ["java-latest-openjdk-headless"], "state": "present"}, + "package": {"name": ["which", "java-latest-openjdk-headless"], "state": "present"}, "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, @@ -56,16 +56,6 @@ class jadx(BaseModule): "remote_src": True, }, }, - { - "name": "Set JAVA_HOME (Fedora)", - "lineinfile": { - "path": "#{BBOT_TOOLS}/jadx/bin/jadx", - "line": "JAVA_HOME=/usr", - "insertafter": "^#!", - }, - "become": True, - "when": "ansible_facts['os_family'] == 'RedHat'", - }, ] allowed_file_types = ["java archive", "android application package"]