From 8a2dd06ce3c2bfba91fc6467c8e36d5f9591ac71 Mon Sep 17 00:00:00 2001 From: Karl Simonsen Date: Tue, 13 Sep 2022 10:49:38 -0700 Subject: [PATCH 1/2] Add bridge file downloader Summary: Add bridge file downloader Purpose is to add an OSS file_downloader.py to match the existing OSS file_uploader.py. Implementation-specific version is not included in OSS. Differential Revision: https://internalfb.com/D39421795 fbshipit-source-id: 59f30919f7cb6a5393ad3fb591a50df97d4a4afb --- .../download_files/file_downloader.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 benchmarking/bridge/file_storage/download_files/file_downloader.py diff --git a/benchmarking/bridge/file_storage/download_files/file_downloader.py b/benchmarking/bridge/file_storage/download_files/file_downloader.py new file mode 100644 index 00000000..e7b3586c --- /dev/null +++ b/benchmarking/bridge/file_storage/download_files/file_downloader.py @@ -0,0 +1,32 @@ +############################################################################## +# Copyright 2022-present, Meta, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +############################################################################## + +download_handles = {} + + +class FileDownloader: + def __init__(self, context: str = "default"): + self.download_handles = getDownloadHandles() + if context not in self.download_handles: + raise RuntimeError(f"No configuration found for {context}") + self.downloader = self.download_handles[context]() + + def downloadFile(self, file, blob=None): + return self.downloader.downloadFile(file, blob=blob) + + def getDownloader(self): + return self.downloader + + +def registerFileDownloader(name, obj): + global download_handles + download_handles[name] = obj + + +def getDownloadHandles(): + return download_handles From 52cef93797af72eb0330823135cfe55007dc4bef Mon Sep 17 00:00:00 2001 From: Karl Simonsen Date: Tue, 13 Sep 2022 10:49:56 -0700 Subject: [PATCH 2/2] Sideloading of perfetto binary Summary: Sideloading of perfetto binary (based on earlier diff D36329256 and leveraging a private build based on D39229401) The motivation here is to build our own copy of the perfetto binary based on the latest OS 12 sources from https://github.com/google/perfetto (OSS). This offers several advantages: 1. Better Perfetto support and fewer errors by using the Android OS 12 version of perfetto even on older systems running older versions of perfetto natively. 2. At least limited perfetto support on some devices not currently running at least Android OS 10 and therefore with no native perfetto binary. 3. Ability to run against a debug version of perfetto for better logging and debugging capabilities. 4. This also gives us our own built traceconv binary which we will probably want to use going forward to better format the output. Limitations: 1. Doesn't really expand memory profiling capability (yet) to all devices. Further work is required to see if further coverage is possible. 2. Doesn't directly overcome any permissions problems related to working on unrooted devices, although in some cases it gets us closer to solving that. 3. Current build includes arm and arm64 flavors only. x86 and x86_64 flavors are also can be built out-of-the-box, but any other variations would take further build configuration work. However, this should cover most of our uncovered devices. Not a panacea, but potentially a big step in the right direction. Reviewed By: axitkhurana Differential Revision: D39291671 fbshipit-source-id: 3a355db9f8bc53431f6f871904a4dba19818ac95 --- .../platforms/android/android_platform.py | 2 +- benchmarking/profilers/perfetto/perfetto.py | 148 ++++++++++++++++-- .../profilers/perfetto/perfetto_config.py | 29 +++- .../test_profiling/test_perfetto_config.py | 6 +- 4 files changed, 161 insertions(+), 24 deletions(-) diff --git a/benchmarking/platforms/android/android_platform.py b/benchmarking/platforms/android/android_platform.py index 88681d5c..ae676296 100644 --- a/benchmarking/platforms/android/android_platform.py +++ b/benchmarking/platforms/android/android_platform.py @@ -266,7 +266,7 @@ def runBinaryBenchmark(self, cmd, *args, **kwargs): elif profiler == "perfetto": if not PerfettoAllSupported(profiling_types): raise BenchmarkArgParseException( - f"Only [{' ,'.join(perfetto_types_supported)}] are supported types for perfetto profiling." + f"Only [{', '.join(perfetto_types_supported)}] are supported types for perfetto profiling." ) # attempt Perfetto profiling, else fallback to standard run return self._runBenchmarkWithPerfetto( diff --git a/benchmarking/profilers/perfetto/perfetto.py b/benchmarking/profilers/perfetto/perfetto.py index 91bacb99..06d46f7c 100644 --- a/benchmarking/profilers/perfetto/perfetto.py +++ b/benchmarking/profilers/perfetto/perfetto.py @@ -18,10 +18,12 @@ from tempfile import NamedTemporaryFile from typing import List +from bridge.file_storage.download_files.file_downloader import FileDownloader from profilers.perfetto.perfetto_config import PerfettoConfig from profilers.profiler_base import ProfilerBase from profilers.utilities import generate_perf_filename, upload_output_files from utils.custom_logger import getLogger +from utils.subprocess_with_logger import processRun from utils.utilities import ( BenchmarkInvalidBinaryException, BenchmarkUnsupportedDeviceException, @@ -34,10 +36,9 @@ It can be used to profile both Android applications and native processes running on Android. It can profile both Java and C++ code on Android. -Perfetto can be used to profile Android benchmarks as both applications and -binaries. The resulting perf data is used to generate an html report -including a flamegraph (TODO). Both perf data and the report are uploaded to manifold -and the urls are returned as a meta dict which can be updated in the benchmark's meta data. +Perfetto can be used to profile Android benchmarks of binaries. The resulting perf data can be opened and interactively viewed using +//https://ui.perfetto.dev/, including a flamegraph (TODO: generate an html report directly). The config file and resulting perfetto +data are uploaded to the cloud. The urls are returned as a dict and added to the benchmark's meta data. """ logger = logging.getLogger(__name__) @@ -94,9 +95,6 @@ def __init__( self.valid = False self.restoreState = False self.perfetto_pid = None - - if self.android_version < 12 and self.options.get("all_heaps", False): - self.options.all_heaps = False self.app_path = _getAppPath(cmd, "program") self.perfetto_config = PerfettoConfig( self.types, self.options, app_name=self.app_path @@ -124,11 +122,26 @@ def __init__( .strip() .lower() ) + + self.perfetto_path = "perfetto" + self.advanced_support = self.android_version >= 12 + self.min_ver = int(self.options.get("min_ver", 11)) + if self.android_version < self.min_ver: + self._init_sideloaded_binary() # updates self.perfetto_path and self.advanced_support if successful + + self.options["all_heaps"] = self.advanced_support and self.options.get( + "all_heaps", False + ) + self.all_heaps_config = ( + " all_heaps: true\n" + if self.options.get("all_heaps", False) + else "" + ) self.perfetto_cmd = [ "cat", self.config_file_device, "|", - "perfetto", + self.perfetto_path, "-d", "--txt", "-c", @@ -144,6 +157,107 @@ def __init__( super(Perfetto, self).__init__(None) + def _init_sideloaded_binary(self): + """ + Verify that perfetto is available on the host, or download it, then copy perfetto onto the mobile device. + This can throw an exception if the perfetto binary is unexpectedly not found on the host machine + and must be caught and logged here. We will then default to the OS installed perfetto or give a version + error if OS version < 10. + + Updates self.perfetto_path and sets self.advanced_support if successful. + """ + binary_folders = { + "armeabi-v7a": "arm", + "arm64-v8a": "arm64", + "x86": "x86", + "x86_64": "x86_64", + } + self.binary_folder = binary_folders[self.platform.platform_abi] + self.user_home = str(Path.home()) + self.host_perfetto_folder = os.path.join(self.user_home, "android/perfetto") + self.host_perfetto_location = os.path.join( + self.host_perfetto_folder, self.binary_folder, "perfetto" + ) + + try: + if self._existsOrDownloadPerfetto(): + self._copyPerfetto() + self.advanced_support = True + except Exception: + getLogger().exception("Perfetto binary could not be copied to the device.") + + def _existsOrDownloadPerfetto(self): + """ + Using an advanced version of the perfetto binary (built from Android OS 12-based sources or better) + via "sideloading" allows us to take advantage of the latest advanced features and bug fixes. + + If a suitable built version of the perfetto OS 12 binary already exists on the host server, use it. + + Otherwise, attempt to download it if possible. + + 1. Only suuported platforms are attempted (currently arm or arm64) + 2. FileDownloader class must have a "default" implementation + + Otherwise, return False and just use the native perfetto binary from the installed device OS. + + An exception will be raised if this should work (i.e., valid platform and implementation) but doesn't. + """ + if not os.path.exists(self.host_perfetto_location): + if self.binary_folder not in ("arm", "arm64"): + # Currently these are the only flavors we support + getLogger().info( + "Cannot download Perfetto.zip: Perfetto.zip doesn't support {self.binary_folder}." + ) + return False + + try: + profiling_files_downloader = FileDownloader("default").getDownloader() + except Exception: + getLogger().exception( + "Cannot download Perfetto.zip: FileDownloader not implemented." + ) + return False + + getLogger().info( + "Perfetto binary cannot be found on the host machine. Attempting to download." + ) + tmpdir = tempfile.mkdtemp() + filename = os.path.join(tmpdir, "perfetto.zip") + profiling_files_downloader.downloadFile(file=filename) + + if not os.path.isdir(self.host_perfetto_folder): + os.makedirs(self.host_perfetto_folder) + output, err = processRun( + ["unzip", "-o", filename, "-d", self.host_perfetto_folder] + ) + if err: + raise RuntimeError( + f"perfetto archive {filename} was not able to be extracted to {self.host_perfetto_folder}" + ) + if not os.path.exists(self.host_perfetto_location): + raise RuntimeError( + f"Perfetto was not extracted to the expected location {self.host_perfetto_location}." + f"Please confirm that it is available for {self.binary_folder}." + ) + + return True + + def _copyPerfetto(self): + """Check if perfetto binary is on device, if not, copy.""" + remote_binary = os.path.join(self.platform.tgt_dir, "perfetto") + if not (self.platform.fileExistsOnPlatform(remote_binary)): + getLogger().info("Copying perfetto to device") + self.platform.copyFilesToPlatform( + self.host_perfetto_location, + target_dir=self.platform.tgt_dir, + copy_files=True, + ) + + # Setup permissions for it, to avoid perfetto call failure + self.adb.shell(["chmod", "777", remote_binary]) + + self.perfetto_path = remote_binary + def __enter__(self): self._start() @@ -153,17 +267,24 @@ def __exit__(self, type, value, traceback): self._finish() def _validate(self): - if self.android_version < 10: + if self.android_version < 10 and not self.advanced_support: raise BenchmarkUnsupportedDeviceException( f"Attempt to run perfetto on {self.platform.type} {self.platform.rel_version} device {self.platform.device_label} ignored." ) if "memory" in self.types: + # perfetto has stopped supporting Android 10 for memory profiling! + if self.android_version < 11 and not self.advanced_support: + raise BenchmarkUnsupportedDeviceException( + f"Attempt to run perfetto memory profiling on {self.platform.type} {self.platform.rel_version} device {self.platform.device_label} ignored." + ) + filename = os.path.basename(self.app_path) if "#" in filename: raise BenchmarkInvalidBinaryException( f"Cannot run perfetto memory profiling on binary filename '{filename}' containing '#'." ) + output = self.adb.shell(["file", self.app_path]) getLogger().info(f"file {self.app_path} returned '{output}'.") @@ -201,10 +322,11 @@ def _start(self): except Exception as e: raise RuntimeError(f"Perfetto profiling failed to start:\n{e}.") else: - if output == 1 or output == [] or output[0] == "1": + if output == 1 or output == [] or output[-1] == "1": raise RuntimeError("Perfetto profiling could not be started.") - self.perfetto_pid = output[0] + # pid is the last "line" of perfetto output (the only line in release builds) + self.perfetto_pid = output[-1] self.valid = True return output @@ -393,7 +515,9 @@ def _setStateForPerfetto(self): def _setupPerfettoConfig( self, ): - config_str = self.perfetto_config.GeneratePerfettoConfig() + config_str = self.perfetto_config.GeneratePerfettoConfig( + advanced_support=self.advanced_support + ) with NamedTemporaryFile() as f: # Write custom perfetto config f.write(config_str.encode("utf-8")) diff --git a/benchmarking/profilers/perfetto/perfetto_config.py b/benchmarking/profilers/perfetto/perfetto_config.py index afdadaf5..11db2049 100644 --- a/benchmarking/profilers/perfetto/perfetto_config.py +++ b/benchmarking/profilers/perfetto/perfetto_config.py @@ -5,11 +5,12 @@ class PerfettoConfig: + ADAPTIVE_SAMPLING_SHMEM_THRESHOLD_DEFAULT = 32746 BUFFER_SIZE_KB_DEFAULT = 256 * 1024 # 256 megabytes BUFFER_SIZE2_KB_DEFAULT = 2 * 1024 # 2 megabytes SHMEM_SIZE_BYTES_DEFAULT = ( - 8192 * 4096 - ) # Shared memory buffer must be a large multiple of 4096 + 16384 * 4096 + ) # Shared memory buffer value must be a large POWER of 2 of at least 4096 SAMPLING_INTERVAL_BYTES_DEFAULT = 4096 DUMP_INTERVAL_MS_DEFAULT = 1000 BATTERY_POLL_MS_DEFAULT = 1000 @@ -27,7 +28,8 @@ def __init__( self.options = options self.app_name = app_name - def GeneratePerfettoConfig(self) -> str: + def GeneratePerfettoConfig(self, *, advanced_support: bool = False) -> str: + """advanced support: Running at least OS 12 version of Perffeto binary""" # Write custom perfetto config android_log_config = "" cpu_scheduling_details_ftrace_config = "" @@ -44,11 +46,6 @@ def GeneratePerfettoConfig(self) -> str: power_ftrace_config = "" power_suspend_resume_config = "" track_event_config = "" - all_heaps_config = ( - " all_heaps: true\n" - if self.options.get("all_heaps", False) - else "" - ) app_name = self.options.get("app_name", self.app_name) buffer_size_kb = self.options.get("buffer_size_kb", self.BUFFER_SIZE_KB_DEFAULT) buffer_size2_kb = self.options.get( @@ -63,6 +60,20 @@ def GeneratePerfettoConfig(self) -> str: shmem_size_bytes = self.options.get( "shmem_size_bytes", self.SHMEM_SIZE_BYTES_DEFAULT ) + adaptive_sampling_shmem_threshold = self.options.get( + "adaptive_sampling_shmem_threshold", + self.ADAPTIVE_SAMPLING_SHMEM_THRESHOLD_DEFAULT, + ) + adaptive_sampling_shmem_threshold_config = ( + f" adaptive_sampling_shmem_threshold: {adaptive_sampling_shmem_threshold}\n" + if advanced_support + else "" + ) + all_heaps_config = ( + " all_heaps: true\n" + if self.options.get("all_heaps", False) + else "" + ) sampling_interval_bytes = self.options.get( "sampling_interval_bytes", self.SAMPLING_INTERVAL_BYTES_DEFAULT ) @@ -73,6 +84,7 @@ def GeneratePerfettoConfig(self) -> str: heapprofd_config = HEAPPROFD_CONFIG.format( all_heaps_config=all_heaps_config, shmem_size_bytes=shmem_size_bytes, + adaptive_sampling_shmem_threshold_config=adaptive_sampling_shmem_threshold_config, sampling_interval_bytes=sampling_interval_bytes, dump_interval_ms=dump_interval_ms, dump_phase_ms=dump_phase_ms, @@ -203,6 +215,7 @@ def GeneratePerfettoConfig(self) -> str: }} process_cmdline: "{app_name}" shmem_size_bytes: {shmem_size_bytes} +{adaptive_sampling_shmem_threshold_config}\ block_client: true {all_heaps_config}\ }} diff --git a/benchmarking/tests/test_profiling/test_perfetto_config.py b/benchmarking/tests/test_profiling/test_perfetto_config.py index 86beeb31..8c43171f 100644 --- a/benchmarking/tests/test_profiling/test_perfetto_config.py +++ b/benchmarking/tests/test_profiling/test_perfetto_config.py @@ -84,7 +84,7 @@ def test_generate_perfetto_config_cpu_gpu_memory(self): dump_interval_ms: 1000 } process_cmdline: "program" - shmem_size_bytes: 33554432 + shmem_size_bytes: 67108864 block_client: true } } @@ -120,7 +120,7 @@ def test_generate_perfetto_config_cpu_gpu_memory(self): dump_interval_ms: 1000 } process_cmdline: "program" - shmem_size_bytes: 33554432 + shmem_size_bytes: 67108864 block_client: true } } @@ -303,7 +303,7 @@ def test_generate_perfetto_config_cpu_gpu_memory(self): dump_interval_ms: 1000 } process_cmdline: "program" - shmem_size_bytes: 33554432 + shmem_size_bytes: 67108864 block_client: true } }