From 04a071130795582e03f4aa6005a1c0169c169d66 Mon Sep 17 00:00:00 2001 From: moz-wptsync-bot Date: Mon, 18 Jun 2018 04:20:48 -0400 Subject: [PATCH] [Gecko Bug 1323620] Add "fennec" product to wptrunner. (#11530) This allows running web-platform-tests on Fennec given a running emulator. (Which is how we expect the tests to run in automation as well -- the android_emulator_unittest mozharness script takes care of emulator start-up.) It also hooks up ./mach wpt. wptrunner sets up a profile for Fennec, forwards the marionette port and starts up Fennec, etc. = Usage = Set your mozconfig to build fennec. Start an emulator: `./mach android-emulator --version x86` Install fennec: `./mach build && ./mach package && ./mach install` Run the tests: ``` ./mach wpt --product=fennec --testtype=testharness --certutil-binary path/to/host/os/certutil path/to/some/tests ``` Differential Revision: https://phabricator.services.mozilla.com/D1587 bugzilla-url: https://bugzilla.mozilla.org/show_bug.cgi?id=1323620 gecko-commit: 266c6cfb96a76daaa7bd2e3574f7d1d8ae60c4a6 gecko-integration-branch: autoland gecko-reviewers: jgraham --- tools/wpt/browser.py | 22 ++ tools/wpt/run.py | 9 + .../wptrunner/wptrunner/browsers/__init__.py | 3 +- tools/wptrunner/wptrunner/browsers/fennec.py | 254 ++++++++++++++++++ tools/wptrunner/wptrunner/browsers/firefox.py | 4 +- tools/wptrunner/wptrunner/wptcommandline.py | 7 +- 6 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 tools/wptrunner/wptrunner/browsers/fennec.py diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py index c350946ea0c682..fd495fd83ac98f 100644 --- a/tools/wpt/browser.py +++ b/tools/wpt/browser.py @@ -283,6 +283,28 @@ def version(self, binary=None): return m.group(1) +class Fennec(Browser): + """Fennec-specific interface.""" + + product = "fennec" + requirements = "requirements_firefox.txt" + + def install(self, dest=None): + raise NotImplementedError + + def find_binary(self, venv_path=None): + raise NotImplementedError + + def find_webdriver(self): + raise NotImplementedError + + def install_webdriver(self, dest=None): + raise NotImplementedError + + def version(self, binary=None): + return None + + class Chrome(Browser): """Chrome-specific interface. diff --git a/tools/wpt/run.py b/tools/wpt/run.py index b51407f36b2a06..ed5c56f5c23eaa 100644 --- a/tools/wpt/run.py +++ b/tools/wpt/run.py @@ -204,6 +204,14 @@ def setup_kwargs(self, kwargs): kwargs["prefs_root"] = prefs_root +class Fennec(BrowserSetup): + name = "fennec" + browser_cls = browser.Fennec + + def setup_kwargs(self, kwargs): + pass + + class Chrome(BrowserSetup): name = "chrome" browser_cls = browser.Chrome @@ -374,6 +382,7 @@ def setup_kwargs(self, kwargs): product_setup = { + "fennec": Fennec, "firefox": Firefox, "chrome": Chrome, "chrome_android": ChromeAndroid, diff --git a/tools/wptrunner/wptrunner/browsers/__init__.py b/tools/wptrunner/wptrunner/browsers/__init__.py index 6f0c49e8b1fa56..d8682e16a551e5 100644 --- a/tools/wptrunner/wptrunner/browsers/__init__.py +++ b/tools/wptrunner/wptrunner/browsers/__init__.py @@ -6,7 +6,7 @@ "browser": String indicating the Browser implementation used to launch that product. "executor": Dictionary with keys as supported test types and values as the name - of the Executor implemantation that will be used to run that test + of the Executor implementation that will be used to run that test type. "browser_kwargs": String naming function that takes product, binary, prefs_root and the wptrunner.run_tests kwargs dict as arguments @@ -25,6 +25,7 @@ product_list = ["chrome", "chrome_android", "edge", + "fennec", "firefox", "ie", "safari", diff --git a/tools/wptrunner/wptrunner/browsers/fennec.py b/tools/wptrunner/wptrunner/browsers/fennec.py new file mode 100644 index 00000000000000..8818760ab36c1d --- /dev/null +++ b/tools/wptrunner/wptrunner/browsers/fennec.py @@ -0,0 +1,254 @@ +import os +import signal +import sys +import tempfile +import traceback + +import moznetwork +from mozprocess import ProcessHandler +from mozprofile import FirefoxProfile +from mozrunner import FennecEmulatorRunner + +from serve.serve import make_hosts_file + +from .base import (get_free_port, + cmd_arg, + browser_command) +from ..executors.executormarionette import MarionetteTestharnessExecutor # noqa: F401 +from .firefox import (get_timeout_multiplier, update_properties, executor_kwargs, FirefoxBrowser) # noqa: F401 + + +__wptrunner__ = {"product": "fennec", + "check_args": "check_args", + "browser": "FennecBrowser", + "executor": {"testharness": "MarionetteTestharnessExecutor"}, + "browser_kwargs": "browser_kwargs", + "executor_kwargs": "executor_kwargs", + "env_extras": "env_extras", + "env_options": "env_options", + "run_info_extras": "run_info_extras", + "update_properties": "update_properties"} + +class FennecProfile(FirefoxProfile): + # WPT-specific prefs are set in FennecBrowser.start() + FirefoxProfile.preferences.update({ + # Make sure Shield doesn't hit the network. + "app.normandy.api_url": "", + # Increase the APZ content response timeout in tests to 1 minute. + "apz.content_response_timeout": 60000, + # Enable output of dump() + "browser.dom.window.dump.enabled": True, + # Disable safebrowsing components + "browser.safebrowsing.blockedURIs.enabled": False, + "browser.safebrowsing.downloads.enabled": False, + "browser.safebrowsing.passwords.enabled": False, + "browser.safebrowsing.malware.enabled": False, + "browser.safebrowsing.phishing.enabled": False, + # Do not restore the last open set of tabs if the browser has crashed + "browser.sessionstore.resume_from_crash": False, + # Disable Android snippets + "browser.snippets.enabled": False, + "browser.snippets.syncPromo.enabled": False, + "browser.snippets.firstrunHomepage.enabled": False, + # Do not allow background tabs to be zombified, otherwise for tests that + # open additional tabs, the test harness tab itself might get unloaded + "browser.tabs.disableBackgroundZombification": True, + # Disable e10s by default + "browser.tabs.remote.autostart": False, + # Don't warn when exiting the browser + "browser.warnOnQuit": False, + # Don't send Firefox health reports to the production server + "datareporting.healthreport.about.reportUrl": "http://%(server)s/dummy/abouthealthreport/", + # Automatically unload beforeunload alerts + "dom.disable_beforeunload": True, + # Disable the ProcessHangMonitor + "dom.ipc.reportProcessHangs": False, + # No slow script dialogs + "dom.max_chrome_script_run_time": 0, + "dom.max_script_run_time": 0, + # Make sure opening about:addons won"t hit the network + "extensions.webservice.discoverURL": "http://%(server)s/dummy/discoveryURL", + # No hang monitor + "hangmonitor.timeout": 0, + + "javascript.options.showInConsole": True, + # Ensure blocklist updates don't hit the network + "services.settings.server": "http://%(server)s/dummy/blocklist/", + # Disable password capture, so that tests that include forms aren"t + # influenced by the presence of the persistent doorhanger notification + "signon.rememberSignons": False, + }) + + +def check_args(**kwargs): + pass + +def browser_kwargs(test_type, run_info_data, **kwargs): + return {"package_name": kwargs["package_name"], + "device_serial": kwargs["device_serial"], + "prefs_root": kwargs["prefs_root"], + "extra_prefs": kwargs["extra_prefs"], + "test_type": test_type, + "debug_info": kwargs["debug_info"], + "symbols_path": kwargs["symbols_path"], + "stackwalk_binary": kwargs["stackwalk_binary"], + "certutil_binary": kwargs["certutil_binary"], + "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(), + "stackfix_dir": kwargs["stackfix_dir"], + "binary_args": kwargs["binary_args"], + "timeout_multiplier": get_timeout_multiplier(test_type, + run_info_data, + **kwargs), + "leak_check": kwargs["leak_check"], + "stylo_threads": kwargs["stylo_threads"], + "chaos_mode_flags": kwargs["chaos_mode_flags"], + "config": kwargs["config"]} + + +def env_extras(**kwargs): + return [] + + +def run_info_extras(**kwargs): + return {"e10s": False, + "headless": False} + + +def env_options(): + # The server host is set to public localhost IP so that resources can be accessed + # from Android emulator + return {"server_host": moznetwork.get_ip(), + "bind_address": False, + "supports_debugger": True} + + +def write_hosts_file(config, device): + new_hosts = make_hosts_file(config, moznetwork.get_ip()) + current_hosts = device.get_file("/etc/hosts") + if new_hosts == current_hosts: + return + hosts_fd, hosts_path = tempfile.mkstemp() + try: + with os.fdopen(hosts_fd, "w") as f: + f.write(new_hosts) + device.remount() + device.push(hosts_path, "/etc/hosts") + finally: + os.remove(hosts_path) + + +class FennecBrowser(FirefoxBrowser): + used_ports = set() + init_timeout = 300 + shutdown_timeout = 60 + + def __init__(self, logger, prefs_root, test_type, package_name=None, + device_serial="emulator-5444", **kwargs): + FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs) + self._package_name = package_name + self.device_serial = device_serial + + @property + def package_name(self): + """ + Name of app to run on emulator. + """ + if self._package_name is None: + self._package_name = "org.mozilla.fennec" + user = os.getenv("USER") + if user: + self._package_name += "_" + user + return self._package_name + + def start(self, **kwargs): + if self.marionette_port is None: + self.marionette_port = get_free_port(2828, exclude=self.used_ports) + self.used_ports.add(self.marionette_port) + + env = {} + env["MOZ_CRASHREPORTER"] = "1" + env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" + env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" + env["STYLO_THREADS"] = str(self.stylo_threads) + if self.chaos_mode_flags is not None: + env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) + + preferences = self.load_prefs() + + self.profile = FennecProfile(preferences=preferences) + self.profile.set_preferences({"marionette.port": self.marionette_port, + "dom.disable_open_during_load": False, + "places.history.enabled": False, + "dom.send_after_paint_to_content": True, + "network.preload": True}) + + if self.leak_check and kwargs.get("check_leaks", True): + self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log") + if os.path.exists(self.leak_report_file): + os.remove(self.leak_report_file) + env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file + else: + self.leak_report_file = None + + if self.ca_certificate_path is not None: + self.setup_ssl() + + debug_args, cmd = browser_command(self.package_name, + self.binary_args if self.binary_args else [] + + [cmd_arg("marionette"), "about:blank"], + self.debug_info) + + self.runner = FennecEmulatorRunner(app=self.package_name, + profile=self.profile, + cmdargs=cmd[1:], + env=env, + symbols_path=self.symbols_path, + serial=self.device_serial, + # TODO - choose appropriate log dir + logdir=os.getcwd(), + process_class=ProcessHandler, + process_args={"processOutputLine": [self.on_output]}) + + self.logger.debug("Starting Fennec") + # connect to a running emulator + self.runner.device.connect() + + write_hosts_file(self.config, self.runner.device.device) + + self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) + + # gecko_log comes from logcat when running with device/emulator + logcat_args = { + "filterspec": "Gecko", + "serial": self.runner.device.app_ctx.device_serial + } + # TODO setting logcat_args["logfile"] yields an almost empty file + # even without filterspec + logcat_args["stream"] = sys.stdout + self.runner.device.start_logcat(**logcat_args) + + self.runner.device.device.forward( + local="tcp:{}".format(self.marionette_port), + remote="tcp:{}".format(self.marionette_port)) + + self.logger.debug("Fennec Started") + + def stop(self, force=False): + if self.runner is not None: + try: + if self.runner.device.connected: + self.runner.device.device.remove_forwards( + "tcp:{}".format(self.marionette_port)) + except Exception: + traceback.print_exception(*sys.exc_info()) + # We assume that stopping the runner prompts the + # browser to shut down. This allows the leak log to be written + for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)), + (False, lambda: self.runner.stop(signal.SIGTERM)), + (False, lambda: self.runner.stop(signal.SIGKILL))]: + if not force or not clean: + retcode = stop_f() + if retcode is not None: + self.logger.info("Browser exited with return code %s" % retcode) + break + self.logger.debug("stopped") diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py index 9f57bb4424dff2..1707f677b819ad 100644 --- a/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/tools/wptrunner/wptrunner/browsers/firefox.py @@ -54,6 +54,8 @@ def get_timeout_multiplier(test_type, run_info_data, **kwargs): return 4 else: return 3 + elif run_info_data["os"] == "android": + return 4 return 1 @@ -369,7 +371,7 @@ def setup_ssl(self): # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() - certutil_dir = os.path.dirname(self.binary) + certutil_dir = os.path.dirname(self.binary or self.certutil_binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: diff --git a/tools/wptrunner/wptrunner/wptcommandline.py b/tools/wptrunner/wptrunner/wptcommandline.py index d26024533c15b0..7e18a8b4379b33 100644 --- a/tools/wptrunner/wptrunner/wptcommandline.py +++ b/tools/wptrunner/wptrunner/wptcommandline.py @@ -169,7 +169,7 @@ def create_parser(product_choices=None): config_group = parser.add_argument_group("Configuration") config_group.add_argument("--binary", action="store", - type=abs_path, help="Binary to run tests against") + type=abs_path, help="Desktop binary to run tests against") config_group.add_argument('--binary-arg', default=[], action="append", dest="binary_args", help="Extra argument for the binary") @@ -178,7 +178,10 @@ def create_parser(product_choices=None): config_group.add_argument('--webdriver-arg', default=[], action="append", dest="webdriver_args", help="Extra argument for the WebDriver binary") - + config_group.add_argument("--package-name", action="store", + help="Android package name to run tests against") + config_group.add_argument("--device-serial", action="store", + help="Running Android instance to connect to, if not emulator-5554") config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to root directory containing test metadata"), config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",