Add taskgraph (#5145)

* taskgraph config setup * add build and docker-image kinds remove build details and add run-task to dockerfile * add android-build dockerfile * use android-build in build kind add fetch kind * update scripts * update android-gradle-dependencies * add lints and register focus taskgraph * use gradlew in build kind * enable linting * Add GithubDetailsTask and register gradle tasks for taskcluster reporting * add tasks to lint kind * add unit test kind * add build transform * add gradle transform * python3 and other fixes change printVariants output to str * tweak build logic to break out klar/focus jobs * add more builds * add ui test kind * reconfig ui tests * add additional attributes * temporarily disable firebase token use * Rework .taskcluster.yml to use taskgraph for push & pull requests, and the existing automation for cron (nightlies) and releases * add more builds for debugging * cleanup, rebase fix and ui-test work * refactor ui-test docker file, kind and scripts update secrets schema to reflect reality * cleanup, add apk artifact template and ui-test changes * add multi-dep transform and loader changes to ui-test config * set up ui-tests like fenix and change ui-test config * remove unneeded file * update build path more changes for ui tests make requested changes to ui docker file removed dead code * fix artifact reference * change build name to include 'public' and fix dependency keys * use test targets from existing flank file * fix androidTest build * add test targets * Use different approach for ui tests * change firebase project * change to json in ui-test kind * update gcloud and GOOGLE PROJECT; clean up ui-test kind * add reporting checks Co-authored-by: Ben Hearsum <[email protected]>
mozilla-mobile · Sep 21, 2021 · e1b3cb5 · e1b3cb5
1 parent a21fcf8
commit e1b3cb5
Show file tree

Hide file tree

Showing 39 changed files with 3,160 additions and 262 deletions.
diff --git a/.taskcluster.yml b/.taskcluster.yml
diff --git a/app/build.gradle b/app/build.gradle
@@ -9,6 +9,7 @@ apply plugin: 'kotlin-android-extensions'
 apply from: "$project.rootDir/tools/gradle/versionCode.gradle"
 
 import com.android.build.OutputFile
+import groovy.json.JsonOutput
 
 android {
     compileSdkVersion 30
@@ -539,3 +540,30 @@ if (gradle.hasProperty('localProperties.autoPublish.application-services.dir'))
     ext.appServicesSrcDir = gradle."localProperties.autoPublish.application-services.dir"
     apply from: "../${appServicesSrcDir}/build-scripts/substitute-local-appservices.gradle"
 }
+
+// -------------------------------------------------------------------------------------------------
+// Task for printing APK information for the requested variant
+// Taskgraph Usage: "./gradlew printVariants
+// -------------------------------------------------------------------------------------------------
+tasks.register('printVariants') {
+    doLast {
+        def variants = android.applicationVariants.collect { variant -> [
+            apks: variant.outputs.collect { output -> [
+                abi: output.getFilter(com.android.build.VariantOutput.FilterType.ABI),
+                fileName: output.outputFile.name
+            ]},
+            build_type: variant.buildType.name,
+            name: variant.name,
+        ]}
+        // AndroidTest is a special case not included above
+        variants.add([
+            apks: [[
+                abi: 'noarch',
+                fileName: 'app-focus-debug-androidTest.apk',
+            ]],
+            build_type: 'debug',
+            name: 'FocusDebugAndroidTest',
+        ])
+        println 'variants: ' + JsonOutput.toJson(variants)
+    }
+}
diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml
@@ -0,0 +1,66 @@
+# gcloud args match the official gcloud cli
+# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+gcloud:
+  results-bucket: focus_android_test_artifacts
+  record-video: true
+
+  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
+  timeout: 30m
+  # will start test then close socket. no reports will be generated.
+  # to retrieve results later, use the "refresh" command
+  # reports will be generated from /results/matrix_ids.json
+  #async: true 
+  # will start test then leave socket open. reports will be published
+  # to /results
+  # see: https://github.com/TestArmada/flank/issues/339
+  async: false
+
+  # results-history-name
+  # by default, set to app name
+  # declare results-history-name to create a separate dropdown menu in Firebase 
+  # see: https://github.com/TestArmada/flank/issues/341
+  #results-history-name: tmp_parallel 
+
+  # The number of times a test execution should be re-attempted if one or more failures occur.
+  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
+  num-flaky-test-attempts: 1
+
+  # test and app are the only required args
+  app: /app/path 
+  test: /test/path
+
+  auto-google-login: true
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+    - package org.mozilla.focus.activity
+    - package org.mozilla.focus.privacy
+    - package org.mozilla.focus.screenshots
+
+  device:
+   - model: Pixel2
+     version: 28
+
+flank:
+  project: GOOGLE_PROJECT
+  # test shards - the amount of groups to split the test suite into
+  # set to -1 to use one shard per test. default: 1
+  max-test-shards: -1
+  # num-test-runs: the amount of times to run the tests.
+  # 1 runs the tests once. 10 runs all the tests 10x
+  num-test-runs: 1
+  ### Output Style flag
+  ## Output style of execution status. May be one of [verbose, multi, single, compact].
+  ## For runs with only one test execution the default value is 'verbose', in other cases
+  ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
+  ## which don't support ansi codes, to avoid corrupted output use single or verbose.
+  ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
+  output-style: compact
+  ### Full Junit Result flag
+  ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
+  full-junit-result: true
diff --git a/automation/taskcluster/androidTest/parse-ui-test.py b/automation/taskcluster/androidTest/parse-ui-test.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import sys
+import argparse
+from pathlib import Path
+import json
+import yaml
+
+def parse_args(cmdln_args):
+    parser = argparse.ArgumentParser(description="Parse UI test logs an results")
+    parser.add_argument(
+        "--output-md",
+        type=argparse.FileType("w", encoding="utf-8"),
+        help="Output markdown file.",
+        required=True,
+    )
+    parser.add_argument(
+        "--log",
+        type=argparse.FileType("r", encoding="utf-8"),
+        help="Log output of flank.",
+        required=True,
+    )
+    parser.add_argument(
+        "--results", type=Path, help="Directory containing flank results", required=True
+    )
+    parser.add_argument(
+        "--exit-code", type=int, help="Exit code of flank.", required=True
+    )
+    parser.add_argument("--device-type", help="Type of device ", required=True)
+    return parser.parse_args(args=cmdln_args)
+
+
+def extract_android_args(log):
+    return yaml.safe_load(log.split("AndroidArgs\n")[1].split("RunTests\n")[0])
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+
+    log = args.log.read()
+    matrix_ids = json.loads(args.results.joinpath("matrix_ids.json").read_text())
+    #with args.results.joinpath("flank.yml") as f:
+    #    flank_config = yaml.safe_load(f)
+
+    android_args = extract_android_args(log)
+
+    print = args.output_md.write
+
+    print("# Devices\n")
+    print(yaml.safe_dump(android_args["gcloud"]["device"]))
+
+    print("# Results\n")
+    print("| matrix | result | logs | details \n")
+    print("| --- | --- | --- | --- |\n")
+    for matrix, matrix_result in matrix_ids.items():
+        print("| {matrixId} | {outcome} | [logs]({webLink}) | {axes[0][details]}\n".format(**matrix_result))
+
+
+if __name__ == "__main__":
+    main()
+
diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script does the following:
+# 1. Retrieves gcloud service account token
+# 2. Activates gcloud service account
+# 3. Connects to google Firebase (using TestArmada's Flank tool)
+# 4. Executes UI tests
+# 5. Puts test artifacts into the test_artifacts folder
+
+# NOTE:
+# Flank supports sharding across multiple devices at a time, but gcloud API
+# only supports 1 defined APK per test run.
+
+
+# If a command fails then do not proceed and fail this script too.
+set -e
+
+#########################
+# The command line help #
+#########################
+display_help() {
+    echo "Usage: $0 Build_Variant [Number_Shards...]"
+    echo
+    echo "Examples:"
+    echo "To run UI tests on ARM device shard (1 test / shard)"
+    echo "$ ui-test.sh arm64-v8a -1"
+    echo
+    echo "To run UI tests on X86 device (on 3 shards)"
+    echo "$ ui-test.sh x86 3"
+    echo
+}
+
+get_abs_filename() {
+  relative_filename="$1"
+  echo "$(cd "$(dirname "$relative_filename")" && pwd)/$(basename "$relative_filename")"
+}
+
+
+# Basic parameter check
+if [[ $# -lt 1 ]]; then
+    echo "Error: please provide at least one build variant (arm|x86)"
+    display_help
+    exit 1
+fi
+
+device_type="$1"  # arm64-v8a | armeabi-v7a | x86_64 | x86
+APK_APP="$2"
+APK_TEST="$3"
+if [[ ! -z "$4" ]]; then
+    num_shards=$4
+fi
+
+JAVA_BIN="/usr/bin/java"
+PATH_TEST="./automation/taskcluster/androidTest"
+FLANK_BIN="/builds/worker/test-tools/flank.jar"
+ARTIFACT_DIR="/builds/worker/artifacts"
+RESULTS_DIR="${ARTIFACT_DIR}/results"
+
+echo
+echo "ACTIVATE SERVICE ACCT"
+echo
+# this is where the Google Testcloud project ID is set
+gcloud config set project "$GOOGLE_PROJECT"
+echo
+
+gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
+echo
+echo
+
+# Disable exiting on error. If the tests fail we want to continue
+# and try to download the artifacts. We will exit with the actual error code later.
+set +e
+
+if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
+    flank_template="${PATH_TEST}/flank-${device_type}.yml"
+elif [[ "${device_type}" == "x86-start-test" ]]; then
+    flank_template="${PATH_TEST}/flank-x86-start-test.yml"
+elif [[ "${device_type}" == "arm-start-test" ]]; then
+    flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml"
+elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then
+    flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml"
+elif [[ "${device_type}" == "x86-beta-tests" ]]; then
+    flank_template="${PATH_TEST}/flank-x86-beta.yml"
+else
+    echo "FAILURE: flank config file not found!"
+    exitcode=1
+fi
+
+APK_APP="$(get_abs_filename $APK_APP)"
+APK_TEST="$(get_abs_filename $APK_TEST)"
+echo "device_type: ${device_type}"
+echo "APK_APP: ${APK_APP}"
+echo "APK_TEST: ${APK_TEST}"
+
+# function to exit script with exit code from test run.
+# (Only 0 if all test executions passed)
+function failure_check() {
+    echo
+    echo
+    if [[ $exitcode -ne 0 ]]; then
+        echo "FAILURE: UI test run failed, please check above URL"
+    else
+	      echo "All UI test(s) have passed!"
+    fi
+
+    echo
+    echo "RESULTS"
+    echo
+
+    mkdir -p /builds/worker/artifacts/github
+    chmod +x $PATH_TEST/parse-ui-test.py
+    $PATH_TEST/parse-ui-test.py \
+        --exit-code "${exitcode}" \
+        --log flank.log \
+        --results "${RESULTS_DIR}" \
+        --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \
+	--device-type "${device_type}"
+}
+
+echo
+echo "FLANK VERSION"
+echo
+$JAVA_BIN -jar $FLANK_BIN --version
+echo
+echo
+
+echo
+echo "EXECUTE TEST(S)"
+echo
+# Note that if --local-results-dir is "results", timestamped sub-directory will
+# contain the results. For any other value, the directory itself will have the results.
+set -o pipefail && $JAVA_BIN -jar $FLANK_BIN android run \
+	--config=$flank_template \
+	--max-test-shards=$num_shards \
+	--app=$APK_APP --test=$APK_TEST \
+	--local-result-dir="${RESULTS_DIR}" \
+	--project=$GOOGLE_PROJECT \
+	| tee flank.log
+
+exitcode=$?
+failure_check
+
+exit $exitcode
diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,7 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 
+import org.mozilla.focus.gradle.tasks.GithubDetailsTask
+
 buildscript {
     ext.espresso_version = '3.1.0-alpha4'
     ext.coroutines_version = '1.4.2'
@@ -98,3 +100,15 @@ task ktlintFormat(type: JavaExec, group: "formatting") {
     main = "com.pinterest.ktlint.Main"
     args "-F", "app/**/*.kt", "!**/build/**/*.kt", "buildSrc/**/*.kt"
 }
+
+tasks.register("githubTestDetails", GithubDetailsTask) {
+    text = "### [Unit Test Results]({reportsUrl}/test/testFocusDebugUnitTest/index.html)"
+}
+
+tasks.register("githubLintDetektDetails", GithubDetailsTask) {
+    text = "### [Detekt Results]({reportsUrl}/detekt.html)"
+}
+
+tasks.register("githubLintAndroidDetails", GithubDetailsTask) {
+    text = "### [Android Lint Results]({reportsUrl}/lint-results-debug.html)"
+}
diff --git a/buildSrc/src/main/java/org/mozilla/focus/gradle/tasks/GithubDetailsTask.kt b/buildSrc/src/main/java/org/mozilla/focus/gradle/tasks/GithubDetailsTask.kt
@@ -0,0 +1,38 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.focus.gradle.tasks
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+import java.io.File
+
+/**
+ * Helper to write to the "customCheckRunText.md" file for Taskcluster.
+ * Taskcluster uses this file to populate the "Details" section in the GitHub Checks panel UI.
+ */
+open class GithubDetailsTask : DefaultTask() {
+
+    /**
+     * Text to display in the Github Checks panel under "Details". Any markdown works here.
+     * The text is written to a markdown file which is used by Taskcluster.
+     * Links are automatically rewritten to point to the correct Taskcluster URL.
+     */
+    @Input
+    var text: String = ""
+
+    private val detailsFile = File("/builds/worker/github/customCheckRunText.md")
+    private val suffix = "\n\n_(404 if compilation failed)_"
+
+    @TaskAction
+    fun writeFile() {
+        val taskId = System.getenv("TASK_ID")
+        val reportsUrl = "https://firefoxci.taskcluster-artifacts.net/$taskId/0/public/reports"
+        val replaced = text.replace("{reportsUrl}", reportsUrl)
+
+        project.mkdir("/builds/worker/github")
+        detailsFile.writeText(replaced + suffix)
+    }
+}