[Gecko Bug 1323620] Add "fennec" product to wptrunner. (#11530)

This allows running web-platform-tests on Fennec given a running emulator.
(Which is how we expect the tests to run in automation as well -- the
android_emulator_unittest mozharness script takes care of emulator
start-up.) It also hooks up ./mach wpt.

wptrunner sets up a profile for Fennec, forwards the marionette port
and starts up Fennec, etc.

= Usage =

Set your mozconfig to build fennec.

Start an emulator: `./mach android-emulator --version x86`
Install fennec: `./mach build && ./mach package && ./mach install`
Run the tests:

```
./mach wpt --product=fennec --testtype=testharness
--certutil-binary path/to/host/os/certutil path/to/some/tests
```

Differential Revision: https://phabricator.services.mozilla.com/D1587

bugzilla-url: https://bugzilla.mozilla.org/show_bug.cgi?id=1323620
gecko-commit: 266c6cfb96a76daaa7bd2e3574f7d1d8ae60c4a6
gecko-integration-branch: autoland
gecko-reviewers: jgraham

tools/wpt/browser.py

@@ -283,6 +283,28 @@ def version(self, binary=None):
         return m.group(1)
 
 
+class Fennec(Browser):
+    """Fennec-specific interface."""
+
+    product = "fennec"
+    requirements = "requirements_firefox.txt"
+
+    def install(self, dest=None):
+        raise NotImplementedError
+
+    def find_binary(self, venv_path=None):
+        raise NotImplementedError
+
+    def find_webdriver(self):
+        raise NotImplementedError
+
+    def install_webdriver(self, dest=None):
+        raise NotImplementedError
+
+    def version(self, binary=None):
+        return None
+
+
 class Chrome(Browser):
     """Chrome-specific interface.
 

tools/wpt/run.py

@@ -204,6 +204,14 @@ def setup_kwargs(self, kwargs):
             kwargs["prefs_root"] = prefs_root
 
 
+class Fennec(BrowserSetup):
+    name = "fennec"
+    browser_cls = browser.Fennec
+
+    def setup_kwargs(self, kwargs):
+        pass
+
+
 class Chrome(BrowserSetup):
     name = "chrome"
     browser_cls = browser.Chrome
@@ -374,6 +382,7 @@ def setup_kwargs(self, kwargs):
 
 
 product_setup = {
+    "fennec": Fennec,
     "firefox": Firefox,
     "chrome": Chrome,
     "chrome_android": ChromeAndroid,

tools/wptrunner/wptrunner/browsers/__init__.py

@@ -6,7 +6,7 @@
 "browser": String indicating the Browser implementation used to launch that
            product.
 "executor": Dictionary with keys as supported test types and values as the name
-            of the Executor implemantation that will be used to run that test
+            of the Executor implementation that will be used to run that test
             type.
 "browser_kwargs": String naming function that takes product, binary,
                   prefs_root and the wptrunner.run_tests kwargs dict as arguments
@@ -25,6 +25,7 @@
 product_list = ["chrome",
                 "chrome_android",
                 "edge",
+                "fennec",
                 "firefox",
                 "ie",
                 "safari",

tools/wptrunner/wptrunner/browsers/fennec.py

@@ -0,0 +1,254 @@
+import os
+import signal
+import sys
+import tempfile
+import traceback
+
+import moznetwork
+from mozprocess import ProcessHandler
+from mozprofile import FirefoxProfile
+from mozrunner import FennecEmulatorRunner
+
+from serve.serve import make_hosts_file
+
+from .base import (get_free_port,
+                   cmd_arg,
+                   browser_command)
+from ..executors.executormarionette import MarionetteTestharnessExecutor  # noqa: F401
+from .firefox import (get_timeout_multiplier, update_properties, executor_kwargs, FirefoxBrowser)  # noqa: F401
+
+
+__wptrunner__ = {"product": "fennec",
+                 "check_args": "check_args",
+                 "browser": "FennecBrowser",
+                 "executor": {"testharness": "MarionetteTestharnessExecutor"},
+                 "browser_kwargs": "browser_kwargs",
+                 "executor_kwargs": "executor_kwargs",
+                 "env_extras": "env_extras",
+                 "env_options": "env_options",
+                 "run_info_extras": "run_info_extras",
+                 "update_properties": "update_properties"}
+
+class FennecProfile(FirefoxProfile):
+    # WPT-specific prefs are set in FennecBrowser.start()
+    FirefoxProfile.preferences.update({
+        # Make sure Shield doesn't hit the network.
+        "app.normandy.api_url": "",
+        # Increase the APZ content response timeout in tests to 1 minute.
+        "apz.content_response_timeout": 60000,
+        # Enable output of dump()
+        "browser.dom.window.dump.enabled": True,
+        # Disable safebrowsing components
+        "browser.safebrowsing.blockedURIs.enabled": False,
+        "browser.safebrowsing.downloads.enabled": False,
+        "browser.safebrowsing.passwords.enabled": False,
+        "browser.safebrowsing.malware.enabled": False,
+        "browser.safebrowsing.phishing.enabled": False,
+        # Do not restore the last open set of tabs if the browser has crashed
+        "browser.sessionstore.resume_from_crash": False,
+        # Disable Android snippets
+        "browser.snippets.enabled": False,
+        "browser.snippets.syncPromo.enabled": False,
+        "browser.snippets.firstrunHomepage.enabled": False,
+        # Do not allow background tabs to be zombified, otherwise for tests that
+        # open additional tabs, the test harness tab itself might get unloaded
+        "browser.tabs.disableBackgroundZombification": True,
+        # Disable e10s by default
+        "browser.tabs.remote.autostart": False,
+        # Don't warn when exiting the browser
+        "browser.warnOnQuit": False,
+        # Don't send Firefox health reports to the production server
+        "datareporting.healthreport.about.reportUrl": "http://%(server)s/dummy/abouthealthreport/",
+        # Automatically unload beforeunload alerts
+        "dom.disable_beforeunload": True,
+        # Disable the ProcessHangMonitor
+        "dom.ipc.reportProcessHangs": False,
+        # No slow script dialogs
+        "dom.max_chrome_script_run_time": 0,
+        "dom.max_script_run_time": 0,
+        # Make sure opening about:addons won"t hit the network
+        "extensions.webservice.discoverURL": "http://%(server)s/dummy/discoveryURL",
+        # No hang monitor
+        "hangmonitor.timeout": 0,
+
+        "javascript.options.showInConsole": True,
+        # Ensure blocklist updates don't hit the network
+        "services.settings.server": "http://%(server)s/dummy/blocklist/",
+        # Disable password capture, so that tests that include forms aren"t
+        # influenced by the presence of the persistent doorhanger notification
+        "signon.rememberSignons": False,
+    })
+
+
+def check_args(**kwargs):
+    pass
+
+def browser_kwargs(test_type, run_info_data, **kwargs):
+    return {"package_name": kwargs["package_name"],
+            "device_serial": kwargs["device_serial"],
+            "prefs_root": kwargs["prefs_root"],
+            "extra_prefs": kwargs["extra_prefs"],
+            "test_type": test_type,
+            "debug_info": kwargs["debug_info"],
+            "symbols_path": kwargs["symbols_path"],
+            "stackwalk_binary": kwargs["stackwalk_binary"],
+            "certutil_binary": kwargs["certutil_binary"],
+            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
+            "stackfix_dir": kwargs["stackfix_dir"],
+            "binary_args": kwargs["binary_args"],
+            "timeout_multiplier": get_timeout_multiplier(test_type,
+                                                         run_info_data,
+                                                         **kwargs),
+            "leak_check": kwargs["leak_check"],
+            "stylo_threads": kwargs["stylo_threads"],
+            "chaos_mode_flags": kwargs["chaos_mode_flags"],
+            "config": kwargs["config"]}
+
+
+def env_extras(**kwargs):
+    return []
+
+
+def run_info_extras(**kwargs):
+    return {"e10s": False,
+            "headless": False}
+
+
+def env_options():
+    # The server host is set to public localhost IP so that resources can be accessed
+    # from Android emulator
+    return {"server_host": moznetwork.get_ip(),
+            "bind_address": False,
+            "supports_debugger": True}
+
+
+def write_hosts_file(config, device):
+    new_hosts = make_hosts_file(config, moznetwork.get_ip())
+    current_hosts = device.get_file("/etc/hosts")
+    if new_hosts == current_hosts:
+        return
+    hosts_fd, hosts_path = tempfile.mkstemp()
+    try:
+        with os.fdopen(hosts_fd, "w") as f:
+            f.write(new_hosts)
+        device.remount()
+        device.push(hosts_path, "/etc/hosts")
+    finally:
+        os.remove(hosts_path)
+
+
+class FennecBrowser(FirefoxBrowser):
+    used_ports = set()
+    init_timeout = 300
+    shutdown_timeout = 60
+
+    def __init__(self, logger, prefs_root, test_type, package_name=None,
+                 device_serial="emulator-5444", **kwargs):
+        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
+        self._package_name = package_name
+        self.device_serial = device_serial
+
+    @property
+    def package_name(self):
+        """
+        Name of app to run on emulator.
+        """
+        if self._package_name is None:
+            self._package_name = "org.mozilla.fennec"
+            user = os.getenv("USER")
+            if user:
+                self._package_name += "_" + user
+        return self._package_name
+
+    def start(self, **kwargs):
+        if self.marionette_port is None:
+            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
+            self.used_ports.add(self.marionette_port)
+
+        env = {}
+        env["MOZ_CRASHREPORTER"] = "1"
+        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
+        env["STYLO_THREADS"] = str(self.stylo_threads)
+        if self.chaos_mode_flags is not None:
+            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
+
+        preferences = self.load_prefs()
+
+        self.profile = FennecProfile(preferences=preferences)
+        self.profile.set_preferences({"marionette.port": self.marionette_port,
+                                      "dom.disable_open_during_load": False,
+                                      "places.history.enabled": False,
+                                      "dom.send_after_paint_to_content": True,
+                                      "network.preload": True})
+
+        if self.leak_check and kwargs.get("check_leaks", True):
+            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
+            if os.path.exists(self.leak_report_file):
+                os.remove(self.leak_report_file)
+            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+        else:
+            self.leak_report_file = None
+
+        if self.ca_certificate_path is not None:
+            self.setup_ssl()
+
+        debug_args, cmd = browser_command(self.package_name,
+                                          self.binary_args if self.binary_args else [] +
+                                          [cmd_arg("marionette"), "about:blank"],
+                                          self.debug_info)
+
+        self.runner = FennecEmulatorRunner(app=self.package_name,
+                                           profile=self.profile,
+                                           cmdargs=cmd[1:],
+                                           env=env,
+                                           symbols_path=self.symbols_path,
+                                           serial=self.device_serial,
+                                           # TODO - choose appropriate log dir
+                                           logdir=os.getcwd(),
+                                           process_class=ProcessHandler,
+                                           process_args={"processOutputLine": [self.on_output]})
+
+        self.logger.debug("Starting Fennec")
+        # connect to a running emulator
+        self.runner.device.connect()
+
+        write_hosts_file(self.config, self.runner.device.device)
+
+        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
+
+        # gecko_log comes from logcat when running with device/emulator
+        logcat_args = {
+            "filterspec": "Gecko",
+            "serial": self.runner.device.app_ctx.device_serial
+        }
+        # TODO setting logcat_args["logfile"] yields an almost empty file
+        # even without filterspec
+        logcat_args["stream"] = sys.stdout
+        self.runner.device.start_logcat(**logcat_args)
+
+        self.runner.device.device.forward(
+            local="tcp:{}".format(self.marionette_port),
+            remote="tcp:{}".format(self.marionette_port))
+
+        self.logger.debug("Fennec Started")
+
+    def stop(self, force=False):
+        if self.runner is not None:
+            try:
+                if self.runner.device.connected:
+                    self.runner.device.device.remove_forwards(
+                        "tcp:{}".format(self.marionette_port))
+            except Exception:
+                traceback.print_exception(*sys.exc_info())
+            # We assume that stopping the runner prompts the
+            # browser to shut down. This allows the leak log to be written
+            for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
+                                  (False, lambda: self.runner.stop(signal.SIGTERM)),
+                                  (False, lambda: self.runner.stop(signal.SIGKILL))]:
+                if not force or not clean:
+                    retcode = stop_f()
+                    if retcode is not None:
+                        self.logger.info("Browser exited with return code %s" % retcode)
+                        break
+        self.logger.debug("stopped")

tools/wptrunner/wptrunner/browsers/firefox.py

@@ -54,6 +54,8 @@ def get_timeout_multiplier(test_type, run_info_data, **kwargs):
             return 4
         else:
             return 3
+    elif run_info_data["os"] == "android":
+        return 4
     return 1
 
 
@@ -369,7 +371,7 @@ def setup_ssl(self):
         # local copy of certutil
         # TODO: Maybe only set this if certutil won't launch?
         env = os.environ.copy()
-        certutil_dir = os.path.dirname(self.binary)
+        certutil_dir = os.path.dirname(self.binary or self.certutil_binary)
         if mozinfo.isMac:
             env_var = "DYLD_LIBRARY_PATH"
         elif mozinfo.isUnix:

tools/wptrunner/wptrunner/wptcommandline.py

@@ -169,7 +169,7 @@ def create_parser(product_choices=None):
 
     config_group = parser.add_argument_group("Configuration")
     config_group.add_argument("--binary", action="store",
-                              type=abs_path, help="Binary to run tests against")
+                              type=abs_path, help="Desktop binary to run tests against")
     config_group.add_argument('--binary-arg',
                               default=[], action="append", dest="binary_args",
                               help="Extra argument for the binary")
@@ -178,7 +178,10 @@ def create_parser(product_choices=None):
     config_group.add_argument('--webdriver-arg',
                               default=[], action="append", dest="webdriver_args",
                               help="Extra argument for the WebDriver binary")
-
+    config_group.add_argument("--package-name", action="store",
+                              help="Android package name to run tests against")
+    config_group.add_argument("--device-serial", action="store",
+                              help="Running Android instance to connect to, if not emulator-5554")
     config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
                               help="Path to root directory containing test metadata"),
     config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",