From d69426ce0e0f38ffe19837746826a388ce9046bb Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Fri, 11 Aug 2023 15:25:57 -0600 Subject: [PATCH 01/13] Fix path to build Android correctly --- fastlane/Fastfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 60d60934c2ba..ec68c74472de 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -17,7 +17,7 @@ platform :android do desc "Generate a new local APK for e2e testing" lane :build_e2e do ENV["ENVFILE"]="tests/e2e/.env.e2e" - ENV["ENTRY_FILE"]="#{Dir.pwd}/../src/libs/E2E/reactNativeLaunchingTest.js" + ENV["ENTRY_FILE"]="../src/libs/E2E/reactNativeLaunchingTest.js" ENV["E2E_TESTING"]="true" gradle( From 85f0d93e444f6437bff64eba4809c66ea7f2ea68 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Fri, 11 Aug 2023 15:31:03 -0600 Subject: [PATCH 02/13] Add `IS_MERGED` output --- .github/actions/javascript/getPullRequestDetails/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/javascript/getPullRequestDetails/action.yml b/.github/actions/javascript/getPullRequestDetails/action.yml index a59cf55bdf9f..6704d5220851 100644 --- a/.github/actions/javascript/getPullRequestDetails/action.yml +++ b/.github/actions/javascript/getPullRequestDetails/action.yml @@ -15,6 +15,8 @@ outputs: description: 'The merge_commit_sha of the given pull request' MERGE_ACTOR: description: 'The actor who merged the pull request' + IS_MERGED: + description: 'True if the pull request is merged' runs: using: 'node16' main: './index.js' From 45dd2b4e1b00bcea3ce29aa659874ea24f40c795 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Fri, 11 Aug 2023 15:35:14 -0600 Subject: [PATCH 03/13] Add two more missing outputs --- .github/actions/javascript/getPullRequestDetails/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/javascript/getPullRequestDetails/action.yml b/.github/actions/javascript/getPullRequestDetails/action.yml index 6704d5220851..ed2c60f018a1 100644 --- a/.github/actions/javascript/getPullRequestDetails/action.yml +++ b/.github/actions/javascript/getPullRequestDetails/action.yml @@ -13,10 +13,14 @@ inputs: outputs: MERGE_COMMIT_SHA: description: 'The merge_commit_sha of the given pull request' + HEAD_COMMIT_SHA: + description: 'The head_commit_sha of the given pull request' MERGE_ACTOR: description: 'The actor who merged the pull request' IS_MERGED: description: 'True if the pull request is merged' + FORKED_REPO_URL: + description: 'Output forked repo URL if PR includes changes from a fork' runs: using: 'node16' main: './index.js' From 20c1e05b2a54b3b9d06124839ecb6c6833e3a7f5 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Fri, 11 Aug 2023 15:40:54 -0600 Subject: [PATCH 04/13] Temp comment out fork checkout --- .github/workflows/e2ePerformanceTests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index fe364b376e3b..548f7e901667 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -81,12 +81,13 @@ jobs: - name: Unmerged PR - Fetch head ref of unmerged PR if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} run: | - if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then - git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} - git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - else +# TODO: UNDO THIS +# if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then +# git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} +# git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 +# else git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - fi +# fi - name: Unmerged PR - Set dummy git credentials before merging if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} From 0cd275462293f0ca4238545fbb8037cc40904e8d Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Fri, 11 Aug 2023 15:41:48 -0600 Subject: [PATCH 05/13] Another temporary fix --- .github/workflows/e2ePerformanceTests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index 548f7e901667..7888c8c47c25 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -81,13 +81,7 @@ jobs: - name: Unmerged PR - Fetch head ref of unmerged PR if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} run: | -# TODO: UNDO THIS -# if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then -# git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} -# git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 -# else git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 -# fi - name: Unmerged PR - Set dummy git credentials before merging if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} From f7f3a45a86728c1cccd49bde0407d05a1f7f1c31 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Tue, 15 Aug 2023 15:45:25 -0600 Subject: [PATCH 06/13] Tweak path --- fastlane/Fastfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/Fastfile b/fastlane/Fastfile index ec68c74472de..5dfccb4d93ec 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -17,7 +17,7 @@ platform :android do desc "Generate a new local APK for e2e testing" lane :build_e2e do ENV["ENVFILE"]="tests/e2e/.env.e2e" - ENV["ENTRY_FILE"]="../src/libs/E2E/reactNativeLaunchingTest.js" + ENV["ENTRY_FILE"]="{Dir.pwd}/src/libs/E2E/reactNativeLaunchingTest.js" ENV["E2E_TESTING"]="true" gradle( From bd51745eb262a09b416b76c2cbc7f451c6edcedd Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Tue, 15 Aug 2023 16:23:32 -0600 Subject: [PATCH 07/13] Try fixing path yet again! --- fastlane/Fastfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 5dfccb4d93ec..cd11c5cf9e98 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -17,7 +17,7 @@ platform :android do desc "Generate a new local APK for e2e testing" lane :build_e2e do ENV["ENVFILE"]="tests/e2e/.env.e2e" - ENV["ENTRY_FILE"]="{Dir.pwd}/src/libs/E2E/reactNativeLaunchingTest.js" + ENV["ENTRY_FILE"]="src/libs/E2E/reactNativeLaunchingTest.js" ENV["E2E_TESTING"]="true" gradle( From 64e4effbd2c266308c09181007cd4d416c84f6e5 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Tue, 15 Aug 2023 16:52:54 -0600 Subject: [PATCH 08/13] Undo test commit --- .github/workflows/e2ePerformanceTests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index 7888c8c47c25..fe364b376e3b 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -81,7 +81,12 @@ jobs: - name: Unmerged PR - Fetch head ref of unmerged PR if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} run: | + if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then + git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} + git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 + else git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 + fi - name: Unmerged PR - Set dummy git credentials before merging if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} From 9debab5bceb6bb778ac7ed83589159096c05dc32 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Tue, 15 Aug 2023 17:14:28 -0600 Subject: [PATCH 09/13] Testing full flow --- .github/workflows/e2ePerformanceTests.yml | 60 +++-------------------- 1 file changed, 6 insertions(+), 54 deletions(-) diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index fe364b376e3b..557af1a04786 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -16,42 +16,6 @@ on: required: true jobs: - buildBaseline: - runs-on: ubuntu-latest-xl - name: Build apk from latest release as a baseline - outputs: - VERSION: ${{ steps.getMostRecentRelease.outputs.VERSION }} - steps: - - uses: actions/checkout@v3 - - - name: Get most recent release version - id: getMostRecentRelease - run: echo "VERSION=$(gh release list --limit 1 | awk '{ print $1 }')" >> "$GITHUB_OUTPUT" - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: Check if there's an existing artifact for this baseline - id: checkForExistingArtifact - uses: xSAVIKx/artifact-exists-action@3c5206b1411c0d2fc0840f56b7140646933d9d6a - with: - name: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }} - - - name: Skip build if there's already an existing artifact for the baseline - if: ${{ fromJSON(steps.checkForExistingArtifact.outputs.exists) }} - run: echo 'APK for baseline ${{ steps.getMostRecentRelease.outputs.VERSION }} already exists, reusing existing build' - - - name: Checkout "Baseline" commit (last release) - if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }} - run: | - git fetch origin tag ${{ steps.getMostRecentRelease.outputs.VERSION }} --no-tags --depth=1 - git switch --detach ${{ steps.getMostRecentRelease.outputs.VERSION }} - - - name: Build APK - if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }} - uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main - with: - ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }} - buildDelta: runs-on: ubuntu-latest-xl name: Build apk from delta ref @@ -81,12 +45,7 @@ jobs: - name: Unmerged PR - Fetch head ref of unmerged PR if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} run: | - if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then - git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} - git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - else git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - fi - name: Unmerged PR - Set dummy git credentials before merging if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} @@ -119,7 +78,7 @@ jobs: runTestsInAWS: runs-on: ubuntu-latest - needs: [buildBaseline, buildDelta] + needs: [buildDelta] name: Run E2E tests in AWS device farm steps: - uses: actions/checkout@v3 @@ -127,25 +86,18 @@ jobs: - name: Make zip directory for everything to send to AWS Device Farm run: mkdir zip - - name: Download baseline APK - uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b - id: downloadBaselineAPK - with: - name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }} - path: zip - - # The downloaded artifact will be a file named "app-e2eRelease.apk" so we have to rename it - - name: Rename baseline APK - run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk" - - name: Download delta APK uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b + id: downloadDeltaAPK with: name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }} path: zip - name: Rename delta APK - run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-compare.apk" + run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk" + + - name: Copy Delta APK as compare for testing + run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-baseline.apk" - name: Copy e2e code into zip folder run: cp -r tests/e2e zip From e1b500b38a023b5c82b99f0adcd0acac5b73b9a2 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Wed, 16 Aug 2023 11:12:06 -0600 Subject: [PATCH 10/13] Add more logs --- src/libs/E2E/tests/openSearchPageTest.e2e.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js index 2f0f72f35bdd..12fcfc3fd1a7 100644 --- a/src/libs/E2E/tests/openSearchPageTest.e2e.js +++ b/src/libs/E2E/tests/openSearchPageTest.e2e.js @@ -7,12 +7,16 @@ import CONST from '../../../CONST'; const test = () => { // check for login (if already logged in the action will simply resolve) + console.debug('[E2E] Logging in for search'); + E2ELogin().then((neededLogin) => { if (neededLogin) { // we don't want to submit the first login to the results return E2EClient.submitTestDone(); } + console.debug('[E2E] Logged in, getting search metrics and submitting them…'); + Performance.subscribeToMeasurements((entry) => { if (entry.name !== CONST.TIMING.SEARCH_RENDER) { return; @@ -21,7 +25,14 @@ const test = () => { E2EClient.submitTestResults({ name: 'Open Search Page TTI', duration: entry.duration, - }).then(E2EClient.submitTestDone); + }) + .then(() => { + console.debug('[E2E] Done with search, exiting…'); + E2EClient.submitTestDone(); + }) + .catch((err) => { + console.debug('[E2E] Error while submitting test results:', err); + }); }); Navigation.navigate(ROUTES.SEARCH); From 6a93aeb4167656de56731b75e20e4acbc114113b Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Wed, 16 Aug 2023 13:10:24 -0600 Subject: [PATCH 11/13] Add more logs --- src/libs/E2E/tests/openSearchPageTest.e2e.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js index 12fcfc3fd1a7..ded45f55fd9c 100644 --- a/src/libs/E2E/tests/openSearchPageTest.e2e.js +++ b/src/libs/E2E/tests/openSearchPageTest.e2e.js @@ -18,10 +18,12 @@ const test = () => { console.debug('[E2E] Logged in, getting search metrics and submitting them…'); Performance.subscribeToMeasurements((entry) => { + console.debug(`[E2E] Entry: ${entry}`); if (entry.name !== CONST.TIMING.SEARCH_RENDER) { return; } + console.debug(`[E2E] Submitting!`); E2EClient.submitTestResults({ name: 'Open Search Page TTI', duration: entry.duration, From 25af1459173556bef2f09ba0c8dc87db9e2fb707 Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Wed, 16 Aug 2023 14:13:20 -0600 Subject: [PATCH 12/13] =?UTF-8?q?Fix=20logs=20=F0=9F=99=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libs/E2E/tests/openSearchPageTest.e2e.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js index ded45f55fd9c..e96d70c95d94 100644 --- a/src/libs/E2E/tests/openSearchPageTest.e2e.js +++ b/src/libs/E2E/tests/openSearchPageTest.e2e.js @@ -18,7 +18,7 @@ const test = () => { console.debug('[E2E] Logged in, getting search metrics and submitting them…'); Performance.subscribeToMeasurements((entry) => { - console.debug(`[E2E] Entry: ${entry}`); + console.debug(`[E2E] Entry: ${JSON.stringify(entry)}`); if (entry.name !== CONST.TIMING.SEARCH_RENDER) { return; } From 028b384dce0f9888e0c20f4fd7a5cc1f6bd5586f Mon Sep 17 00:00:00 2001 From: Andrew Gable Date: Wed, 16 Aug 2023 15:48:21 -0600 Subject: [PATCH 13/13] Only navigate to search bar once sidebar is loaded --- src/libs/E2E/tests/openSearchPageTest.e2e.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js index e96d70c95d94..3b2d91322cf0 100644 --- a/src/libs/E2E/tests/openSearchPageTest.e2e.js +++ b/src/libs/E2E/tests/openSearchPageTest.e2e.js @@ -18,6 +18,12 @@ const test = () => { console.debug('[E2E] Logged in, getting search metrics and submitting them…'); Performance.subscribeToMeasurements((entry) => { + if (entry.name === CONST.TIMING.SIDEBAR_LOADED) { + console.debug(`[E2E] Sidebar loaded, navigating to search route…`); + Navigation.navigate(ROUTES.SEARCH); + return; + } + console.debug(`[E2E] Entry: ${JSON.stringify(entry)}`); if (entry.name !== CONST.TIMING.SEARCH_RENDER) { return; @@ -36,8 +42,6 @@ const test = () => { console.debug('[E2E] Error while submitting test results:', err); }); }); - - Navigation.navigate(ROUTES.SEARCH); }); };