From 071bacbec4665f4fae5e373d7852ffcca5811de5 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 13:14:12 +0100 Subject: [PATCH 1/9] Move sleep to beginning of each test --- tests/e2e/testRunner.js | 17 ++++++++--------- tests/e2e/utils/sleep.js | 5 +++++ 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/utils/sleep.js diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 9bdbdfe8efe8..b1d5fb030786 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -27,6 +27,7 @@ const math = require('./measure/math'); const writeTestStats = require('./measure/writeTestStats'); const withFailTimeout = require('./utils/withFailTimeout'); const reversePort = require('./utils/androidReversePort'); +const sleep = require('./utils/sleep'); // VARIABLE CONFIGURATION const args = process.argv.slice(2); @@ -202,6 +203,13 @@ const runTests = async () => { } } + const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`); + coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`); + + // eslint-disable-next-line no-loop-func + await sleep(config.COOL_DOWN); + coolDownLogs.done(); + server.setTestConfig(testConfig); const warmupLogs = Logger.progressInfo(`Running warmup '${testConfig.name}'`); @@ -250,15 +258,6 @@ const runTests = async () => { } } testLog.done(); - - // If we still have tests add a cool down period - if (testIndex < numOfTests - 1) { - const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`); - coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`); - // eslint-disable-next-line no-loop-func - await new Promise((resolve) => setTimeout(resolve, config.COOL_DOWN)); - coolDownLogs.done(); - } } // Calculate statistics and write them to our work file diff --git a/tests/e2e/utils/sleep.js b/tests/e2e/utils/sleep.js new file mode 100644 index 000000000000..6cc4f3bb89d1 --- /dev/null +++ b/tests/e2e/utils/sleep.js @@ -0,0 +1,5 @@ +module.exports = function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +}; From 0682682433485c5f35358e33df2181a2bca904d4 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 13:19:05 +0100 Subject: [PATCH 2/9] Add cooldown between tests --- tests/e2e/config.js | 5 ++++- tests/e2e/testRunner.js | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/e2e/config.js b/tests/e2e/config.js index dae35321fda3..5a879e18afc0 100644 --- a/tests/e2e/config.js +++ b/tests/e2e/config.js @@ -47,7 +47,10 @@ module.exports = { INTERACTION_TIMEOUT: 300000, // Period we wait between each test runs, to let the device cool down - COOL_DOWN: 90 * 1000, + BOOT_COOL_DOWN: 90 * 1000, + + // Period we wait between each test runs, to let the device cool down + SUITE_COOL_DOWN: 5 * 1000, TEST_NAMES, diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index b1d5fb030786..2df80fa97a5b 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -207,7 +207,7 @@ const runTests = async () => { coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`); // eslint-disable-next-line no-loop-func - await sleep(config.COOL_DOWN); + await sleep(config.BOOT_COOL_DOWN); coolDownLogs.done(); server.setTestConfig(testConfig); @@ -237,6 +237,9 @@ const runTests = async () => { progressText = `Suite '${testConfig.name}' [${testIndex + 1}/${numOfTests}], iteration [${i + 1}/${config.RUNS}]\n`; testLog.updateText(progressText); + testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s`); + await sleep(config.SUITE_COOL_DOWN); + await restartApp(); // Wait for a test to finish by waiting on its done call to the http server From f8acc3cec72354e82aa70b74e05a92f7bcbac587 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 13:55:12 +0100 Subject: [PATCH 3/9] Allow sleep between runs --- tests/e2e/testRunner.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 2df80fa97a5b..1464cf2cd90f 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -237,10 +237,14 @@ const runTests = async () => { progressText = `Suite '${testConfig.name}' [${testIndex + 1}/${numOfTests}], iteration [${i + 1}/${config.RUNS}]\n`; testLog.updateText(progressText); + Logger.log('Killing app...'); + await killApp('android', config.APP_PACKAGE); + testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s`); await sleep(config.SUITE_COOL_DOWN); - await restartApp(); + Logger.log('Starting app...'); + await launchApp('android', config.APP_PACKAGE); // Wait for a test to finish by waiting on its done call to the http server try { From 5da00d1ba36519e62adc7764acc4c1d934df26a5 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 15:37:46 +0100 Subject: [PATCH 4/9] Modify runs and cooldowns --- tests/e2e/config.js | 4 ++-- tests/e2e/testRunner.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/config.js b/tests/e2e/config.js index 5a879e18afc0..1e73aa58d3d9 100644 --- a/tests/e2e/config.js +++ b/tests/e2e/config.js @@ -31,7 +31,7 @@ module.exports = { SERVER_PORT: 4723, // The amount of times a test should be executed for average performance metrics - RUNS: 80, + RUNS: 60, DEFAULT_BASELINE_BRANCH: 'main', @@ -50,7 +50,7 @@ module.exports = { BOOT_COOL_DOWN: 90 * 1000, // Period we wait between each test runs, to let the device cool down - SUITE_COOL_DOWN: 5 * 1000, + SUITE_COOL_DOWN: 10 * 1000, TEST_NAMES, diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 1464cf2cd90f..d5be0ebab77f 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -240,7 +240,7 @@ const runTests = async () => { Logger.log('Killing app...'); await killApp('android', config.APP_PACKAGE); - testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s`); + testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s\n`); await sleep(config.SUITE_COOL_DOWN); Logger.log('Starting app...'); From 9a8800db6e1f5c83150245e51ccbf2f79c69f2c8 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 15:48:37 +0100 Subject: [PATCH 5/9] Remove ESLINT comment --- package.json | 2 +- tests/e2e/testRunner.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 3eefcc2cc3a4..a08b63b7bbe9 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production", "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map", "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map", - "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout", + "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout --skipInstallDeps", "test:e2e:delta": "node tests/e2e/testRunner.js --development --branch main --label delta --skipCheckout", "test:e2e:compare": "node tests/e2e/merge.js", "gh-actions-unused-styles": "./.github/scripts/findUnusedKeys.sh", diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index d5be0ebab77f..4d4ac6cd0a22 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -206,7 +206,6 @@ const runTests = async () => { const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`); coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`); - // eslint-disable-next-line no-loop-func await sleep(config.BOOT_COOL_DOWN); coolDownLogs.done(); From 0bf5289f17f5f94b605b93441f1ab9561288eaf6 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 15:54:49 +0100 Subject: [PATCH 6/9] Remove modified package script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a08b63b7bbe9..3eefcc2cc3a4 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production", "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map", "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map", - "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout --skipInstallDeps", + "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout", "test:e2e:delta": "node tests/e2e/testRunner.js --development --branch main --label delta --skipCheckout", "test:e2e:compare": "node tests/e2e/merge.js", "gh-actions-unused-styles": "./.github/scripts/findUnusedKeys.sh", From 2af3d61c06ee867e8e4d315372349c6b77cf80a5 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 16:02:43 +0100 Subject: [PATCH 7/9] Add code comments --- tests/e2e/testRunner.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 4d4ac6cd0a22..0eed35c664de 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -206,6 +206,9 @@ const runTests = async () => { const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`); coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`); + // Having the cooldown right at the beginning should hopefully lower the chances of heat + // throttling from the previous run (which we have no control over and will be a + // completely different AWS DF customer/app). It also gives the time to cool down between test suites. await sleep(config.BOOT_COOL_DOWN); coolDownLogs.done(); @@ -240,6 +243,9 @@ const runTests = async () => { await killApp('android', config.APP_PACKAGE); testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s\n`); + + // Adding the cool down between booting the app again, had the side-effect of actually causing a cold boot, + // which increased TTI/bundle load JS times significantly but also stabilized standard deviation. await sleep(config.SUITE_COOL_DOWN); Logger.log('Starting app...'); From c6adba2a12c4bf7fc458e7d85fb9700ac6d87bc3 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 31 Oct 2023 17:48:44 +0100 Subject: [PATCH 8/9] Increate timeout to AWS DF action --- .github/workflows/e2ePerformanceTests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index ff888c135be9..e1bb286179cf 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -183,6 +183,7 @@ jobs: file_artifacts: Customer Artifacts.zip log_artifacts: debug.log cleanup: true + timeout: 5400 - name: Print logs if run failed if: failure() @@ -213,6 +214,7 @@ jobs: remote_src: false file_artifacts: Customer Artifacts.zip cleanup: true + timeout: 5400 - name: Unzip AWS Device Farm delta results run: unzip "Customer Artifacts.zip" -d deltaResults From 4a2e3eb943844b69551df66251f0cd3554bc87f8 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Wed, 1 Nov 2023 09:37:06 +0100 Subject: [PATCH 9/9] Update tests/e2e/testRunner.js Co-authored-by: Aldo Canepa Garay <87341702+aldo-expensify@users.noreply.github.com> --- tests/e2e/testRunner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 0eed35c664de..d8e4afd606ac 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -208,7 +208,7 @@ const runTests = async () => { // Having the cooldown right at the beginning should hopefully lower the chances of heat // throttling from the previous run (which we have no control over and will be a - // completely different AWS DF customer/app). It also gives the time to cool down between test suites. + // completely different AWS DF customer/app). It also gives the time to cool down between test suites. await sleep(config.BOOT_COOL_DOWN); coolDownLogs.done();