e2e: allow tests failure

Expensify · Oct 22, 2024 · d2c4a0d · d2c4a0d
1 parent ca942ca
commit d2c4a0d
Show file tree

Hide file tree

Showing 10 changed files with 170 additions and 87 deletions.
diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml
@@ -221,6 +221,36 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
+      - name: Check if test has skipped tests
+        id: checkIfSkippedTestsDetected
+        run: |
+          if grep -q '⚠️' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then
+            # Create an output to the GH action that the tests were skipped:
+            echo "skippedTestsDetected=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skippedTestsDetected=false" >> "$GITHUB_OUTPUT"
+            echo '✅ no skipped tests detected'
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: 'Announce skipped tests in Slack'
+        if: ${{ steps.checkIfSkippedTestsDetected.outputs.skippedTestsDetected == 'true' }}
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          custom_payload: |
+            {
+              channel: '#e2e-announce',
+              attachments: [{
+                color: 'danger',
+                text: `⚠️ ${process.env.AS_REPO} Some of E2E tests were skipped on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow ⚠️`,
+              }]
+            }
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+
       - name: 'Announce regression in Slack'
         if: ${{ steps.checkIfRegressionDetected.outputs.performanceRegressionDetected == 'true' }}
         uses: 8398a7/action-slack@v3

diff --git a/src/libs/E2E/tests/appStartTimeTest.e2e.ts b/src/libs/E2E/tests/appStartTimeTest.e2e.ts
@@ -1,11 +1,14 @@
 import Config from 'react-native-config';
+import type {NativeConfig} from 'react-native-config';
 import type {PerformanceEntry} from 'react-native-performance';
 import E2ELogin from '@libs/E2E/actions/e2eLogin';
 import waitForAppLoaded from '@libs/E2E/actions/waitForAppLoaded';
 import E2EClient from '@libs/E2E/client';
+import getConfigValueOrThrow from '@libs/E2E/utils/getConfigValueOrThrow';
 import Performance from '@libs/Performance';
 
-const test = () => {
+const test = (config: NativeConfig) => {
+    const name = getConfigValueOrThrow('name', config);
     // check for login (if already logged in the action will simply resolve)
     E2ELogin().then((neededLogin) => {
         if (neededLogin) {
@@ -25,7 +28,7 @@ const test = () => {
             metrics.map((metric) =>
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: `App start ${metric.name}`,
+                    name: `${name} ${metric.name}`,
                     metric: metric.duration,
                     unit: 'ms',
                 }),

diff --git a/src/libs/E2E/tests/chatOpeningTest.e2e.ts b/src/libs/E2E/tests/chatOpeningTest.e2e.ts
@@ -15,6 +15,7 @@ const test = (config: NativeConfig) => {
     console.debug('[E2E] Logging in for chat opening');
 
     const reportID = getConfigValueOrThrow('reportID', config);
+    const name = getConfigValueOrThrow('name', config);
 
     E2ELogin().then((neededLogin) => {
         if (neededLogin) {
@@ -48,7 +49,7 @@ const test = (config: NativeConfig) => {
             if (entry.name === CONST.TIMING.CHAT_RENDER) {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Chat opening',
+                    name: `${name} Chat opening`,
                     metric: entry.duration,
                     unit: 'ms',
                 })
@@ -64,7 +65,7 @@ const test = (config: NativeConfig) => {
             if (entry.name === CONST.TIMING.OPEN_REPORT) {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Chat TTI',
+                    name: `${name} Chat TTI`,
                     metric: entry.duration,
                     unit: 'ms',
                 })

diff --git a/src/libs/E2E/tests/linkingTest.e2e.ts b/src/libs/E2E/tests/linkingTest.e2e.ts
@@ -24,6 +24,7 @@ const test = (config: NativeConfig) => {
     const reportID = getConfigValueOrThrow('reportID', config);
     const linkedReportID = getConfigValueOrThrow('linkedReportID', config);
     const linkedReportActionID = getConfigValueOrThrow('linkedReportActionID', config);
+    const name = getConfigValueOrThrow('name', config);
 
     E2ELogin().then((neededLogin) => {
         if (neededLogin) {
@@ -74,7 +75,7 @@ const test = (config: NativeConfig) => {
 
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Comment linking',
+                    name,
                     metric: entry.duration,
                     unit: 'ms',
                 });

diff --git a/src/libs/E2E/tests/openSearchRouterTest.e2e.ts b/src/libs/E2E/tests/openSearchRouterTest.e2e.ts
@@ -1,16 +1,20 @@
+import type {NativeConfig} from 'react-native-config';
 import Config from 'react-native-config';
 import * as E2EGenericPressableWrapper from '@components/Pressable/GenericPressable/index.e2e';
 import E2ELogin from '@libs/E2E/actions/e2eLogin';
 import waitForAppLoaded from '@libs/E2E/actions/waitForAppLoaded';
 import E2EClient from '@libs/E2E/client';
+import getConfigValueOrThrow from '@libs/E2E/utils/getConfigValueOrThrow';
 import getPromiseWithResolve from '@libs/E2E/utils/getPromiseWithResolve';
 import Performance from '@libs/Performance';
 import CONST from '@src/CONST';
 
-const test = () => {
+const test = (config: NativeConfig) => {
     // check for login (if already logged in the action will simply resolve)
     console.debug('[E2E] Logging in for new search router');
 
+    const name = getConfigValueOrThrow('name', config);
+
     E2ELogin().then((neededLogin: boolean): Promise<Response> | undefined => {
         if (neededLogin) {
             return waitForAppLoaded().then(() =>
@@ -59,7 +63,7 @@ const test = () => {
             if (entry.name === CONST.TIMING.SEARCH_ROUTER_RENDER) {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Open Search Router TTI',
+                    name: `${name} Open Search Router TTI`,
                     metric: entry.duration,
                     unit: 'ms',
                 })
@@ -75,7 +79,7 @@ const test = () => {
             if (entry.name === CONST.TIMING.LOAD_SEARCH_OPTIONS) {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Load Search Options',
+                    name: `${name} Load Search Options`,
                     metric: entry.duration,
                     unit: 'ms',
                 })

diff --git a/src/libs/E2E/tests/reportTypingTest.e2e.ts b/src/libs/E2E/tests/reportTypingTest.e2e.ts
@@ -21,6 +21,7 @@ const test = (config: NativeConfig) => {
 
     const reportID = getConfigValueOrThrow('reportID', config);
     const message = getConfigValueOrThrow('message', config);
+    const name = getConfigValueOrThrow('name', config);
 
     E2ELogin().then((neededLogin) => {
         if (neededLogin) {
@@ -45,7 +46,7 @@ const test = (config: NativeConfig) => {
             if (entry.name === CONST.TIMING.MESSAGE_SENT) {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
-                    name: 'Message sent',
+                    name: `${name} Message sent`,
                     metric: entry.duration,
                     unit: 'ms',
                 }).then(messageSentResolve);
@@ -77,7 +78,7 @@ const test = (config: NativeConfig) => {
 
                                     E2EClient.submitTestResults({
                                         branch: Config.E2E_BRANCH,
-                                        name: 'Composer typing rerender count',
+                                        name: `${name} Composer typing rerender count`,
                                         metric: rerenderCount,
                                         unit: 'renders',
                                     })

diff --git a/tests/e2e/compare/compare.ts b/tests/e2e/compare/compare.ts
@@ -91,16 +91,23 @@ function compareResults(baselineEntries: Metric | string, compareEntries: Metric
     };
 }
 
-export default (main: Metric | string, delta: Metric | string, outputFile: string, outputFormat = 'all', metricForTest = {}) => {
+type Options = {
+    outputFile: string;
+    outputFormat: 'console' | 'markdown' | 'all';
+    metricForTest: Record<string, Unit>;
+    hasMissingData: boolean;
+};
+
+export default (main: Metric | string, delta: Metric | string, {outputFile, outputFormat = 'all', metricForTest = {}, hasMissingData}: Options) => {
     // IMPORTANT NOTE: make sure you are passing the main/baseline results first, then the delta/compare results:
     const outputData = compareResults(main, delta, metricForTest);
 
     if (outputFormat === 'console' || outputFormat === 'all') {
-        printToConsole(outputData);
+        printToConsole(outputData, hasMissingData);
     }
 
     if (outputFormat === 'markdown' || outputFormat === 'all') {
-        return writeToMarkdown(outputFile, outputData);
+        return writeToMarkdown(outputFile, outputData, hasMissingData);
     }
 };
 export {compareResults};
diff --git a/tests/e2e/compare/output/console.ts b/tests/e2e/compare/output/console.ts
@@ -26,7 +26,7 @@ const printRegularLine = (entry: Entry) => {
 /**
  * Prints the result simply to console.
  */
-export default (data: Data) => {
+export default (data: Data, hasMissingData: boolean) => {
     // No need to log errors or warnings as these were be logged on the fly
     console.debug('');
     console.debug('❇️  Performance comparison results:');
@@ -38,6 +38,10 @@ export default (data: Data) => {
     data.meaningless.forEach(printRegularLine);
 
     console.debug('');
+
+    if (hasMissingData) {
+        console.debug('⚠️ Some tests did not pass successfully, so some results are omitted from final report');
+    }
 };
 
 export type {Data, Entry};
diff --git a/tests/e2e/compare/output/markdown.ts b/tests/e2e/compare/output/markdown.ts
@@ -67,7 +67,7 @@ const buildSummaryTable = (entries: Entry[], collapse = false) => {
     return collapse ? collapsibleSection('Show entries', content) : content;
 };
 
-const buildMarkdown = (data: Data) => {
+const buildMarkdown = (data: Data, hasMissingData: boolean) => {
     let result = '## Performance Comparison Report 📊';
 
     if (data.errors?.length) {
@@ -92,6 +92,10 @@ const buildMarkdown = (data: Data) => {
     result += `\n${buildDetailsTable(data.meaningless)}`;
     result += '\n';
 
+    if (hasMissingData) {
+        result += '⚠️ Some tests did not pass successfully, so some results are omitted from final report';
+    }
+
     return result;
 };
 
@@ -109,8 +113,8 @@ const writeToFile = (filePath: string, content: string) =>
             throw error;
         });
 
-const writeToMarkdown = (filePath: string, data: Data) => {
-    const markdown = buildMarkdown(data);
+const writeToMarkdown = (filePath: string, data: Data, hasMissingData: boolean) => {
+    const markdown = buildMarkdown(data, hasMissingData);
     return writeToFile(filePath, markdown).catch((error) => {
         console.error(error);
         throw error;