e2ePerformanceTests.yml

name: E2E Performance Tests

on:
  workflow_call:
    inputs:
      PR_NUMBER:
        description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
        type: string
        required: true

  workflow_dispatch:
    inputs:
      PR_NUMBER:
        description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
        type: string
        required: true

jobs:
  buildBaseline:
    runs-on: ubuntu-latest-xl
    name: Build apk from latest release as a baseline
    outputs:
      VERSION: ${{ steps.getMostRecentRelease.outputs.VERSION }}
    steps:
      - uses: actions/checkout@v3

      - name: Get most recent release version
        id: getMostRecentRelease
        run: echo "VERSION=$(gh release list --limit 1 | awk '{ print $1 }')" >> "$GITHUB_OUTPUT"
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Check if there's an existing artifact for this baseline
        id: checkForExistingArtifact
        uses: xSAVIKx/artifact-exists-action@3c5206b1411c0d2fc0840f56b7140646933d9d6a
        with:
          name: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}

      - name: Skip build if there's already an existing artifact for the baseline
        if: ${{ fromJSON(steps.checkForExistingArtifact.outputs.exists) }}
        run: echo 'APK for baseline ${{ steps.getMostRecentRelease.outputs.VERSION }} already exists, reusing existing build'

      - name: Checkout "Baseline" commit (last release)
        if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }}
        run: |
          git fetch origin tag ${{ steps.getMostRecentRelease.outputs.VERSION }} --no-tags --depth=1
          git switch --detach ${{ steps.getMostRecentRelease.outputs.VERSION }}

      - name: Configure MapBox SDK
        run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}

      - name: Build APK
        if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }}
        uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main
        with:
          ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }}

  buildDelta:
    runs-on: ubuntu-latest-xl
    name: Build apk from delta ref
    outputs:
      DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }}
    steps:
      - uses: actions/checkout@v3

      - name: Get pull request details
        id: getPullRequestDetails
        uses: Expensify/App/.github/actions/javascript/getPullRequestDetails@main
        with:
          GITHUB_TOKEN: ${{ github.token }}
          PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }}
          USER: ${{ github.actor }}

      - name: Merged PR - Get merge commit sha for the pull request
        if: ${{ fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
        id: getMergeCommitShaIfMergedPR
        run: |
          MERGE_COMMIT_SHA=${{ steps.getPullRequestDetails.outputs.MERGE_COMMIT_SHA }}
          git fetch origin "$MERGE_COMMIT_SHA" --no-tags --depth=1
          echo "MERGE_COMMIT_SHA=$MERGE_COMMIT_SHA" >> "$GITHUB_OUTPUT"
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Unmerged PR - Fetch head ref of unmerged PR
        if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
        run: |
          git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1

      - name: Unmerged PR - Set dummy git credentials before merging
        if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
        run: |
          git config --global user.email "test@test.com"
          git config --global user.name "Test"

      - name: Unmerged PR - Merge pull request locally and get merge commit sha
        if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }}
        id: getMergeCommitShaIfUnmergedPR
        run: |
          git merge --allow-unrelated-histories --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
          git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Determine "delta ref"
        id: getDeltaRef
        run: echo "DELTA_REF=${{ steps.getMergeCommitShaIfMergedPR.outputs.MERGE_COMMIT_SHA || steps.getMergeCommitShaIfUnmergedPR.outputs.MERGE_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Checkout "delta ref"
        run: git checkout ${{ steps.getDeltaRef.outputs.DELTA_REF }}

      - name: Configure MapBox SDK
        run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }}

      - name: Build APK
        uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main
        with:
          ARTIFACT_NAME: delta-apk-${{ steps.getDeltaRef.outputs.DELTA_REF }}

  runTestsInAWS:
    runs-on: ubuntu-latest
    needs: [buildBaseline, buildDelta]
    name: Run E2E tests in AWS device farm
    steps:
      - uses: actions/checkout@v3

      - name: Use Node.js 16.x
        uses: actions/setup-node@v3
        with:
          node-version: 16.x

      - name: Make zip directory for everything to send to AWS Device Farm
        run: mkdir zip

      - name: Download baseline APK
        uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
        id: downloadBaselineAPK
        with:
          name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }}
          path: zip

      # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
      - name: Rename baseline APK
        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk"

      - name: Download delta APK
        uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
        id: downloadDeltaAPK
        with:
          name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }}
          path: zip

      - name: Rename delta APK
        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk"

      - name: Copy e2e code into zip folder
        run: cp -r tests/e2e zip

      - name: Zip everything in the zip directory up
        run: zip -qr App.zip ./zip

      - name: Configure AWS Credentials
        uses: Expensify/App/.github/actions/composite/configureAwsCredentials@main
        with:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: us-west-2

      - name: Schedule AWS Device Farm test run on main branch
        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
        with:
          name: App E2E Performance Regression Tests
          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
          app_file: zip/app-e2eRelease-main.apk
          app_type: ANDROID_APP
          test_type: APPIUM_NODE
          test_package_file: App.zip
          test_package_type: APPIUM_NODE_TEST_PACKAGE
          test_spec_file: tests/e2e/TestSpecMain.yml
          test_spec_type: APPIUM_NODE_TEST_SPEC
          remote_src: false
          file_artifacts: MainArtifacts.zip
          cleanup: true

      - name: Unzip AWS Device Farm main results
        run: unzip MainArtifacts.zip -d mainResults
      
      - name: Schedule AWS Device Farm test run on delta branch
        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
        with:
          name: App E2E Performance Regression Tests
          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
          app_file: zip/app-e2eRelease-delta.apk
          app_type: ANDROID_APP
          test_type: APPIUM_NODE
          test_package_file: App.zip
          test_package_type: APPIUM_NODE_TEST_PACKAGE
          test_spec_file: tests/e2e/TestSpecDelta.yml
          test_spec_type: APPIUM_NODE_TEST_SPEC
          remote_src: false
          file_artifacts: DeltaArtifacts.zip
          cleanup: true

      - name: Unzip AWS Device Farm delta results
        run: unzip DeltaArtifacts.zip -d deltaResults
        
      - name: Compare results
        run: node "./Host_Machine_File/tests/e2e/merge.js --mainPath ../../mainResults/\$WORKING_DIRECTORY/main.json --deltaPath ../../deltaResults/\$WORKING_DIRECTORY/delta.json --outputPath ./Host_Machine_Files/output.md"

      - name: Print results
        run: cat "./Host_Machine_Files/output.md"
        
      # - name: Print AWS Device Farm verbose run results
      #   if: ${{ always() && runner.debug != null && fromJSON(runner.debug) }}
      #   run: cat "./Host_Machine_Files/debug.log"

# TODO: Once tests are more reliable we should uncomment this
#      - name: Check if test failed, if so post the results and add the DeployBlocker label
#        run: |
#          if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then
#            gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
#            gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md
#            gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
#          else
#            echo '✅ no performance regression detected'
#          fi
#        env:
#          GITHUB_TOKEN: ${{ github.token }}