Merge pull request #30793 from margelo/osp/e2e-staggered

[NoQA] Staggered approach to E2E tests
Expensify · Nov 9, 2023 · feae468 · feae468
2 parents 16e40c9 + c2f29a8
commit feae468
Show file tree

Hide file tree

Showing 19 changed files with 511 additions and 310 deletions.
diff --git a/.github/actions/composite/buildAndroidAPKDelta/action.yml b/.github/actions/composite/buildAndroidAPKDelta/action.yml
@@ -0,0 +1,29 @@
+name: Build an Android apk
+description: Build an Android apk for an E2E test build and upload it as an artifact
+
+inputs:
+  ARTIFACT_NAME:
+    description: The name of the workflow artifact where the APK should be uploaded
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: Expensify/App/.github/actions/composite/setupNode@main
+
+    - uses: ruby/setup-ruby@a05e47355e80e57b9a67566a813648fa67d92011
+      with:
+        ruby-version: "2.7"
+        bundler-cache: true
+
+    - uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef
+
+    - name: Build APK
+      run: npm run android-build-e2edelta
+      shell: bash
+
+    - name: Upload APK
+      uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
+      with:
+        name: ${{ inputs.ARTIFACT_NAME }}
+        path: android/app/build/outputs/apk/e2e/release/app-e2edelta-release.apk
diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml
@@ -46,14 +46,28 @@ jobs:
           git fetch origin tag ${{ steps.getMostRecentRelease.outputs.VERSION }} --no-tags --depth=1
           git switch --detach ${{ steps.getMostRecentRelease.outputs.VERSION }}
 
+      - uses: Expensify/App/.github/actions/composite/setupNode@main
+
+      - uses: ruby/setup-ruby@a05e47355e80e57b9a67566a813648fa67d92011
+        with:
+          ruby-version: "2.7"
+          bundler-cache: true
+
+      - uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef
+
       - name: Configure MapBox SDK
         run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}
 
       - name: Build APK
-        if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }}
-        uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main
+        run: npm run android-build-e2e
+        shell: bash
+
+      - name: Upload APK
+        uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
         with:
-          ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}
+          name: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}
+          path: android/app/build/outputs/apk/e2e/release/app-e2e-release.apk
+
 
   buildDelta:
     runs-on: ubuntu-latest-xl
@@ -113,10 +127,24 @@ jobs:
       - name: Configure MapBox SDK
         run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}
 
+      - uses: Expensify/App/.github/actions/composite/setupNode@main
+
+      - uses: ruby/setup-ruby@a05e47355e80e57b9a67566a813648fa67d92011
+        with:
+          ruby-version: "2.7"
+          bundler-cache: true
+
+      - uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef
+
       - name: Build APK
-        uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main
+        run: npm run android-build-e2edelta
+        shell: bash
+
+      - name: Upload APK
+        uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
         with:
-          ARTIFACT_NAME: delta-apk-${{ steps.getDeltaRef.outputs.DELTA_REF }}
+          name: delta-apk-${{ steps.getDeltaRef.outputs.DELTA_REF }}
+          path: android/app/build/outputs/apk/e2edelta/release/app-e2edelta-release.apk
 
   runTestsInAWS:
     runs-on: ubuntu-latest
@@ -140,7 +168,7 @@ jobs:
 
       # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
       - name: Rename baseline APK
-        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk"
+        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk"
 
       - name: Download delta APK
         uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
@@ -150,7 +178,7 @@ jobs:
           path: zip
 
       - name: Rename delta APK
-        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk"
+        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2edelta-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2edeltaRelease.apk"
 
       - name: Copy e2e code into zip folder
         run: cp -r tests/e2e zip
@@ -172,12 +200,12 @@ jobs:
           name: App E2E Performance Regression Tests
           project_arn: ${{ secrets.AWS_PROJECT_ARN }}
           device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
-          app_file: zip/app-e2eRelease-main.apk
+          app_file: zip/app-e2eRelease.apk
           app_type: ANDROID_APP
           test_type: APPIUM_NODE
           test_package_file: App.zip
           test_package_type: APPIUM_NODE_TEST_PACKAGE
-          test_spec_file: tests/e2e/TestSpecMain.yml
+          test_spec_file: tests/e2e/TestSpec.yml
           test_spec_type: APPIUM_NODE_TEST_SPEC
           remote_src: false
           file_artifacts: Customer Artifacts.zip
@@ -192,38 +220,13 @@ jobs:
           unzip "Customer Artifacts.zip" -d mainResults
           cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log
 
-      - name: Unzip AWS Device Farm main results
-        run: unzip "Customer Artifacts.zip" -d mainResults
-
-      - name: Delete Customer Artifacts.zip
-        run: rm "Customer Artifacts.zip"
-
-      - name: Schedule AWS Device Farm test run on delta branch
-        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
-        with:
-          name: App E2E Performance Regression Tests
-          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
-          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
-          app_file: zip/app-e2eRelease-delta.apk
-          app_type: ANDROID_APP
-          test_type: APPIUM_NODE
-          test_package_file: App.zip
-          test_package_type: APPIUM_NODE_TEST_PACKAGE
-          test_spec_file: tests/e2e/TestSpecDelta.yml
-          test_spec_type: APPIUM_NODE_TEST_SPEC
-          remote_src: false
-          file_artifacts: Customer Artifacts.zip
-          cleanup: true
-          timeout: 5400
-
-      - name: Unzip AWS Device Farm delta results
-        run: unzip "Customer Artifacts.zip" -d deltaResults
-
-      - name: Compare results
-        run: node tests/e2e/merge.js --mainPath ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/main.json --deltaPath ./deltaResults//Host_Machine_Files/\$WORKING_DIRECTORY/delta.json --outputPath ./output.md
+      - name: Unzip AWS Device Farm results
+        if: ${{ always() }}
+        run: unzip "Customer Artifacts.zip"
 
-      - name: Print results
-        run: cat "./output.md"
+      - name: Print AWS Device Farm run results
+        if: ${{ always() }}
+        run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
 
       - name: Check if test failed, if so post the results and add the DeployBlocker label
         run: |

diff --git a/android/app/build.gradle b/android/app/build.gradle
@@ -58,7 +58,8 @@ project.ext.envConfigFiles = [
     adhocRelease: ".env.adhoc",
     developmentRelease: ".env",
     developmentDebug: ".env",
-    e2eRelease: "tests/e2e/.env.e2e"
+    e2eRelease: "tests/e2e/.env.e2e",
+    e2edeltaRelease: "tests/e2e/.env.e2edelta"
 ]
 
 /**
@@ -101,7 +102,14 @@ android {
         e2e {
             // If  are building a version that won't be uploaded to the play store, we don't have to use production keys
             // applies all non-production flavors
-            applicationIdSuffix ".adhoc"
+            applicationIdSuffix ".e2e"
+            signingConfig signingConfigs.debug
+            resValue "string", "build_config_package", "com.expensify.chat"
+        }
+        e2edelta {
+            // If  are building a version that won't be uploaded to the play store, we don't have to use production keys
+            // applies all non-production flavors
+            applicationIdSuffix ".e2edelta"
             signingConfig signingConfigs.debug
             resValue "string", "build_config_package", "com.expensify.chat"
         }
@@ -150,12 +158,13 @@ android {
             }
             // ... except for the e2e flavor, which we maybe want to build locally:
             productFlavors.e2e.signingConfig signingConfigs.debug
+            productFlavors.e2edelta.signingConfig signingConfigs.debug
         }
     }
 
     // since we don't need variants adhocDebug and e2eDebug, we can force gradle to ignore them
     variantFilter { variant ->
-        if (variant.name == "adhocDebug" || variant.name == "e2eDebug") {
+        if (variant.name == "adhocDebug" || variant.name == "e2eDebug" || variant.name == "e2edeltaDebug") {
             setIgnore(true)
         }
     }