Merge pull request #28109 from Expensify/andrew-fix-e2e-fork

[No QA]Fix E2E Performance Regression Tests
Expensify · Sep 28, 2023 · da07655 · da07655
2 parents bfda42d + 28260ae
commit da07655
Show file tree

Hide file tree

Showing 18 changed files with 200 additions and 59 deletions.
diff --git a/.github/actions/composite/buildAndroidAPK/action.yml b/.github/actions/composite/buildAndroidAPK/action.yml
@@ -13,7 +13,7 @@ runs:
 
     - uses: ruby/setup-ruby@eae47962baca661befdfd24e4d6c34ade04858f7
       with:
-        ruby-version: '2.7'
+        ruby-version: "2.7"
         bundler-cache: true
 
     - uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef
@@ -26,4 +26,4 @@ runs:
       uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
       with:
         name: ${{ inputs.ARTIFACT_NAME }}
-        path: android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk
+        path: android/app/build/outputs/apk/e2e/release/app-e2e-release.apk
diff --git a/.github/actions/javascript/getPullRequestDetails/action.yml b/.github/actions/javascript/getPullRequestDetails/action.yml
@@ -13,8 +13,14 @@ inputs:
 outputs:
   MERGE_COMMIT_SHA:
     description: 'The merge_commit_sha of the given pull request'
+  HEAD_COMMIT_SHA:
+    description: 'The head_commit_sha of the given pull request'
   MERGE_ACTOR:
     description: 'The actor who merged the pull request'
+  IS_MERGED:
+    description: 'True if the pull request is merged'
+  FORKED_REPO_URL:
+    description: 'Output forked repo URL if PR includes changes from a fork'
 runs:
   using: 'node16'
   main: './index.js'
diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml
@@ -84,12 +84,7 @@ jobs:
       - name: Unmerged PR - Fetch head ref of unmerged PR
         if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
         run: |
-          if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then
-            git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }}
-            git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1
-          else
-            git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1
-          fi
+          git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1
 
       - name: Unmerged PR - Set dummy git credentials before merging
         if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
@@ -101,7 +96,7 @@ jobs:
         if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
         id: getMergeCommitShaIfUnmergedPR
         run: |
-          git merge --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
+          git merge --allow-unrelated-histories --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
           git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
         env:
           GITHUB_TOKEN: ${{ github.token }}
@@ -140,18 +135,19 @@ jobs:
           name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }}
           path: zip
 
-      # The downloaded artifact will be a file named "app-e2eRelease.apk" so we have to rename it
+      # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
       - name: Rename baseline APK
-        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk"
+        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk"
 
       - name: Download delta APK
         uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
+        id: downloadDeltaAPK
         with:
           name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }}
           path: zip
 
       - name: Rename delta APK
-        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-compare.apk"
+        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk"
 
       - name: Copy e2e code into zip folder
         run: cp -r tests/e2e zip

diff --git a/android/app/build.gradle b/android/app/build.gradle
@@ -58,7 +58,7 @@ project.ext.envConfigFiles = [
     adhocRelease: ".env.adhoc",
     developmentRelease: ".env",
     developmentDebug: ".env",
-    e2eRelease: ".env.production"
+    e2eRelease: "tests/e2e/.env.e2e"
 ]
 
 /**
@@ -136,10 +136,20 @@ android {
             signingConfig signingConfigs.debug
         }
         release {
-            signingConfig signingConfigs.release
             productFlavors.production.signingConfig signingConfigs.release
             minifyEnabled enableProguardInReleaseBuilds
             proguardFiles getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro"
+
+            signingConfig null
+            // buildTypes take precedence over productFlavors when it comes to the signing configuration,
+            // thus we need to manually set the signing config, so that the e2e uses the debug config again.
+            // In other words, the signingConfig setting above will be ignored when we build the flavor in release mode. 
+            productFlavors.all { flavor ->
+                // All release builds should be signed with the release config ...
+                flavor.signingConfig signingConfigs.release
+            }
+            // ... except for the e2e flavor, which we maybe want to build locally:
+            productFlavors.e2e.signingConfig signingConfigs.debug
         }
     }
 

diff --git a/fastlane/Fastfile b/fastlane/Fastfile
@@ -17,7 +17,7 @@ platform :android do
   desc "Generate a new local APK for e2e testing"
   lane :build_e2e do
     ENV["ENVFILE"]="tests/e2e/.env.e2e"
-    ENV["ENTRY_FILE"]="#{Dir.pwd}/../src/libs/E2E/reactNativeLaunchingTest.js"
+    ENV["ENTRY_FILE"]="src/libs/E2E/reactNativeLaunchingTest.js"
     ENV["E2E_TESTING"]="true"
 
     gradle(

diff --git a/scripts/android-repackage-app-bundle-and-sign.sh b/scripts/android-repackage-app-bundle-and-sign.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+source ./scripts/shellUtils.sh
 
 ###
 # Takes an android app that has been built with the debug keystore,
@@ -41,7 +42,7 @@ if [ ! -f "$NEW_BUNDLE_FILE" ]; then
     echo "Bundle file not found: $NEW_BUNDLE_FILE"
     exit 1
 fi
-OUTPUT_APK=$(realpath "$OUTPUT_APK")
+OUTPUT_APK=$(get_abs_path "$OUTPUT_APK")
 # check if "apktool" command is available
 if ! command -v apktool &> /dev/null
 then

diff --git a/scripts/shellUtils.sh b/scripts/shellUtils.sh
@@ -41,3 +41,46 @@ function join_by_string {
   shift
   printf "%s" "$first" "${@/#/$separator}"
 }
+
+# Usage: get_abs_path <path>
+# Will make a path absolute, resolving any relative paths
+# example: get_abs_path "./foo/bar"
+get_abs_path() {
+    local the_path=$1
+    local -a path_elements
+    IFS='/' read -ra path_elements <<< "$the_path"
+
+    # If the path is already absolute, start with an empty string.
+    # We'll prepend the / later when reconstructing the path.
+    if [[ "$the_path" = /* ]]; then
+        abs_path=""
+    else
+        abs_path="$(pwd)"
+    fi
+
+    # Handle each path element
+    for element in "${path_elements[@]}"; do
+        if [ "$element" = "." ] || [ -z "$element" ]; then
+            continue
+        elif [ "$element" = ".." ]; then
+            # Remove the last element from abs_path
+            abs_path=$(dirname "$abs_path")
+        else
+            # Append element to the absolute path
+            abs_path="${abs_path}/${element}"
+        fi
+    done
+
+    # Remove any trailing '/'
+    while [[ $abs_path == */ ]]; do
+        abs_path=${abs_path%/}
+    done
+
+    # Special case for root
+    [ -z "$abs_path" ] && abs_path="/"
+
+    # Special case to remove any starting '//' when the input path was absolute
+    abs_path=${abs_path/#\/\//\/}
+
+    echo "$abs_path"
+}
diff --git a/src/libs/E2E/API.mock.js b/src/libs/E2E/API.mock.js
@@ -19,6 +19,7 @@ const mocks = {
     BeginSignIn: mockBeginSignin,
     SigninUser: mockSigninUser,
     OpenApp: mockOpenApp,
+    ReconnectApp: mockOpenApp,
     OpenReport: mockOpenReport,
     AuthenticatePusher: mockAuthenticatePusher,
 };

diff --git a/src/libs/E2E/reactNativeLaunchingTest.js b/src/libs/E2E/reactNativeLaunchingTest.js
@@ -6,11 +6,7 @@
  */
 
 import Performance from '../Performance';
-
-// start the usual app
-Performance.markStart('regularAppStart');
-import '../../../index';
-Performance.markEnd('regularAppStart');
+import * as Metrics from '../Metrics';
 
 import E2EConfig from '../../../tests/e2e/config';
 import E2EClient from './client';
@@ -19,6 +15,11 @@ console.debug('==========================');
 console.debug('==== Running e2e test ====');
 console.debug('==========================');
 
+// Check if the performance module is available
+if (!Metrics.canCapturePerformanceMetrics()) {
+    throw new Error('Performance module not available! Please set CAPTURE_METRICS=true in your environment file!');
+}
+
 // import your test here, define its name and config first in e2e/config.js
 const tests = {
     [E2EConfig.TEST_NAMES.AppStartTime]: require('./tests/appStartTimeTest.e2e').default,
@@ -36,20 +37,33 @@ const appReady = new Promise((resolve) => {
     });
 });
 
-E2EClient.getTestConfig().then((config) => {
-    const test = tests[config.name];
-    if (!test) {
-        // instead of throwing, report the error to the server, which is better for DX
-        return E2EClient.submitTestResults({
-            name: config.name,
-            error: `Test '${config.name}' not found`,
-        });
-    }
-    console.debug(`[E2E] Configured for test ${config.name}. Waiting for app to become ready`);
-
-    appReady.then(() => {
-        console.debug('[E2E] App is ready, running test…');
-        Performance.measureFailSafe('appStartedToReady', 'regularAppStart');
-        test();
+E2EClient.getTestConfig()
+    .then((config) => {
+        const test = tests[config.name];
+        if (!test) {
+            // instead of throwing, report the error to the server, which is better for DX
+            return E2EClient.submitTestResults({
+                name: config.name,
+                error: `Test '${config.name}' not found`,
+            });
+        }
+
+        console.debug(`[E2E] Configured for test ${config.name}. Waiting for app to become ready`);
+        appReady
+            .then(() => {
+                console.debug('[E2E] App is ready, running test…');
+                Performance.measureFailSafe('appStartedToReady', 'regularAppStart');
+                test();
+            })
+            .catch((error) => {
+                console.error('[E2E] Error while waiting for app to become ready', error);
+            });
+    })
+    .catch((error) => {
+        console.error("[E2E] Error while running test. Couldn't get test config!", error);
     });
-});
+
+// start the usual app
+Performance.markStart('regularAppStart');
+import '../../../index';
+Performance.markEnd('regularAppStart');
diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js
@@ -7,24 +7,41 @@ import CONST from '../../../CONST';
 
 const test = () => {
     // check for login (if already logged in the action will simply resolve)
+    console.debug('[E2E] Logging in for search');
+
     E2ELogin().then((neededLogin) => {
         if (neededLogin) {
             // we don't want to submit the first login to the results
             return E2EClient.submitTestDone();
         }
 
+        console.debug('[E2E] Logged in, getting search metrics and submitting them…');
+
         Performance.subscribeToMeasurements((entry) => {
+            if (entry.name === CONST.TIMING.SIDEBAR_LOADED) {
+                console.debug(`[E2E] Sidebar loaded, navigating to search route…`);
+                Navigation.navigate(ROUTES.SEARCH);
+                return;
+            }
+
+            console.debug(`[E2E] Entry: ${JSON.stringify(entry)}`);
             if (entry.name !== CONST.TIMING.SEARCH_RENDER) {
                 return;
             }
 
+            console.debug(`[E2E] Submitting!`);
             E2EClient.submitTestResults({
                 name: 'Open Search Page TTI',
                 duration: entry.duration,
-            }).then(E2EClient.submitTestDone);
+            })
+                .then(() => {
+                    console.debug('[E2E] Done with search, exiting…');
+                    E2EClient.submitTestDone();
+                })
+                .catch((err) => {
+                    console.debug('[E2E] Error while submitting test results:', err);
+                });
         });
-
-        Navigation.navigate(ROUTES.SEARCH);
     });
 };
 

diff --git a/tests/e2e/ADDING_TESTS.md b/tests/e2e/ADDING_TESTS.md
@@ -1,4 +1,51 @@
-# Add E2E Tests
+# Adding new E2E Tests
+
+## Running your new test in development mode
+
+Typically you'd run all the tests with `npm run test:e2e` on your machine,
+this will run the tests with some local settings, however that is not
+optimal when you add a new test for which you want to quickly test if it works, as it
+still runs the release version of the app.
+
+I recommend doing the following.
+
+> [!NOTE]
+> All of the steps can be executed at once by running XXX (todo)
+
+1. Rename `./index.js` to `./appIndex.js`
+2. Create a new `./index.js` with the following content:
+```js
+requrire("./src/libs/E2E/reactNativeLaunchingTest.js");
+```
+3. In `./src/libs/E2E/reactNativeLaunchingTest.js` change the main app import to the new `./appIndex.js` file:
+```diff
+- import '../../../index';
++ import '../../../appIndex';
+```
+
+> [!WARNING]
+> Make sure to not commit these changes to the repository!
+
+Now you can start the metro bundler in e2e mode with:
+
+```
+CAPTURE_METRICS=TRUE E2E_Testing=true npm start -- --reset-cache
+```
+
+Then we can execute our test with:
+
+```
+npm run test:e2e -- --development --skipInstallDeps --buildMode skip --includes "My new test name"
+```
+
+> - `--development` will run the tests with a local config, which will run the tests with fewer iterations
+> - `--skipInstallDeps` will skip the `npm install` step, which you probably don't need
+> - `--buildMode skip` will skip rebuilding the app, and just run the existing app
+> - `--includes "MyTestName"` will only run the test with the name "MyTestName"
+
+
+
+## Creating a new test
 
 Tests are executed on device, inside the app code.
 
@@ -97,6 +144,10 @@ Done! When you now start the test runner, your new test will be executed as well
 ## Quickly test your test
 
 To check your new test you can simply run `npm run test:e2e`, which uses the
-`--development` flag. This will run the tests on the branch you are currently on
-and will do fewer iterations.
+`--development` flag. This will run the tests on the branch you are currently on, runs fewer iterations and most importantly, it tries to reuse the existing APK and just patch into the new app bundle, instead of rebuilding the release app from scratch.
+
+## Debugging your test
+
+You can use regular console statements to debug your test. The output will be visible
+in logcat. I recommend opening the android studio logcat window and filter for `ReactNativeJS` to see the output you'd otherwise typically see in your metro bundler instance.
 
diff --git a/tests/e2e/config.js b/tests/e2e/config.js
@@ -21,7 +21,7 @@ const TEST_NAMES = {
  * ```
  */
 module.exports = {
-    APP_PACKAGE: 'com.expensify.chat',
+    APP_PACKAGE: 'com.expensify.chat.adhoc',
 
     APP_PATHS: {
         baseline: './app-e2eRelease-baseline.apk',

diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js
@@ -1,8 +1,10 @@
 module.exports = {
+    APP_PACKAGE: 'com.expensify.chat.dev',
+
     WARM_UP_RUNS: 1,
     RUNS: 8,
     APP_PATHS: {
-        baseline: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk',
-        compare: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk',
+        baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
+        compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
     },
 };