Skip to content

E2E Performance Tests #106

E2E Performance Tests

E2E Performance Tests #106

name: E2E Performance Tests
on:
workflow_call:
inputs:
PR_NUMBER:
description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
type: string
required: true
workflow_dispatch:
inputs:
PR_NUMBER:
description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
type: string
required: true
concurrency:
group: ${{ github.ref == 'refs/heads/main' && format('{0}-{1}', github.ref, github.sha) || github.ref }}-e2e
cancel-in-progress: true
jobs:
prep:
runs-on: ubuntu-latest
name: Find the baseline and delta refs, and check for an existing build artifact for that commit
outputs:
BASELINE_ARTIFACT_FOUND: ${{ steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND }}
BASELINE_ARTIFACT_WORKFLOW_ID: ${{ steps.checkForExistingArtifact.outputs.ARTIFACT_WORKFLOW_ID }}
BASELINE_VERSION: ${{ steps.getMostRecentRelease.outputs.VERSION }}
DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }}
IS_PR_MERGED: ${{ steps.getPullRequestDetails.outputs.IS_MERGED }}
steps:
- uses: actions/checkout@v4
with:
# The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API)
token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
- name: Get most recent release version
id: getMostRecentRelease
run: echo "VERSION=$(gh release list --limit 1 | awk '{ print $1 }')" >> "$GITHUB_OUTPUT"
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Check if there's an existing artifact for this baseline
id: checkForExistingArtifact
uses: ./.github/actions/javascript/getArtifactInfo
with:
GITHUB_TOKEN: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
ARTIFACT_NAME: baseline-${{ steps.getMostRecentRelease.outputs.VERSION }}-android-artifact-apk
- name: Skip build if there's already an existing artifact for the baseline
if: ${{ fromJSON(steps.checkForExistingArtifact.outputs.ARTIFACT_FOUND) }}
run: echo 'APK for baseline ${{ steps.getMostRecentRelease.outputs.VERSION }} already exists, reusing existing build'
- name: Get pull request details
id: getPullRequestDetails
uses: ./.github/actions/javascript/getPullRequestDetails
with:
GITHUB_TOKEN: ${{ github.token }}
PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }}
USER: ${{ github.actor }}
- name: Determine "delta ref"
id: getDeltaRef
run: |
if [ '${{ steps.getPullRequestDetails.outputs.IS_MERGED }}' == 'true' ]; then
echo "DELTA_REF=${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
else
# Set dummy git credentials
git config --global user.email "[email protected]"
git config --global user.name "Test"
# Fetch head_ref of unmerged PR
git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1
# Merge pull request locally and get merge commit sha
git merge --allow-unrelated-histories -X ours --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
# Create and push a branch so it can be checked out in another runner
git checkout -b e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
git push origin e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
echo "DELTA_REF=e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
fi
buildBaseline:
name: Build apk from latest release as a baseline
uses: ./.github/workflows/buildAndroid.yml
needs: prep
if: ${{ fromJSON(needs.prep.outputs.BASELINE_ARTIFACT_FOUND) }}
secrets: inherit
with:
type: e2e
ref: ${{ needs.prep.outputs.BASELINE_VERSION }}
artifact-prefix: baseline-${{ needs.prep.outputs.BASELINE_VERSION }}
buildDelta:
name: Build apk from delta ref
uses: ./.github/workflows/buildAndroid.yml
needs: prep
secrets: inherit
with:
type: e2eDelta
ref: ${{ needs.prep.outputs.DELTA_REF }}
artifact-prefix: delta-${{ needs.prep.outputs.DELTA_REF }}
runTestsInAWS:
runs-on: ubuntu-latest
needs: [prep, buildBaseline, buildDelta]
name: Run E2E tests in AWS device farm
steps:
- uses: actions/checkout@v4
with:
# The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API)
token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}
- name: Setup Node
uses: ./.github/actions/composite/setupNode
- name: Make zip directory for everything to send to AWS Device Farm
run: mkdir zip
- name: Download baseline APK
uses: actions/download-artifact@v4
id: downloadBaselineAPK
with:
name: baseline-${{ needs.prep.outputs.BASELINE_VERSION }}-android-apk
path: zip
# Set github-token only if the baseline was built in this workflow run:
github-token: ${{ needs.prep.outputs.BASELINE_ARTIFACT_WORKFLOW_ID && github.token }}
run-id: ${{ needs.prep.outputs.BASELINE_ARTIFACT_WORKFLOW_ID }}
# The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
- name: Rename baseline APK
run: mv "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2e-release.apk" "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2eRelease.apk"
- name: Download delta APK
uses: actions/download-artifact@v4
id: downloadDeltaAPK
with:
name: delta-${{ needs.prep.outputs.DELTA_REF }}-android-apk
path: zip
- name: Rename delta APK
run: mv "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edelta-release.apk" "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edeltaRelease.apk"
- name: Compile test runner to be executable in a nodeJS environment
run: npm run e2e-test-runner-build
- name: Copy e2e code into zip folder
run: cp tests/e2e/dist/index.js zip/testRunner.ts
- name: Copy profiler binaries into zip folder
run: cp -r node_modules/@perf-profiler/android/cpp-profiler/bin zip/bin
- name: Zip everything in the zip directory up
run: zip -qr App.zip ./zip
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Schedule AWS Device Farm test run on main branch
uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
id: schedule-awsdf-main
with:
name: App E2E Performance Regression Tests
project_arn: ${{ secrets.AWS_PROJECT_ARN }}
device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
app_file: zip/app-e2eRelease.apk
app_type: ANDROID_APP
test_type: APPIUM_NODE
test_package_file: App.zip
test_package_type: APPIUM_NODE_TEST_PACKAGE
test_spec_file: tests/e2e/TestSpec.yml
test_spec_type: APPIUM_NODE_TEST_SPEC
remote_src: false
file_artifacts: |
Customer Artifacts.zip
Test spec output.txt
log_artifacts: debug.log
cleanup: true
timeout: 7200
- name: Print logs if run failed
if: failure()
run: |
echo ${{ steps.schedule-awsdf-main.outputs.data }}
unzip "Customer Artifacts.zip" -d mainResults
cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/logcat.txt" || true
cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log || true
cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/Test spec output.txt" || true
- name: Announce failed workflow in Slack
if: failure()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
channel: '#e2e-announce',
attachments: [{
color: 'danger',
text: `💥 ${process.env.AS_REPO} E2E Test run failed on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow 💥`,
}]
}
env:
GITHUB_TOKEN: ${{ github.token }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
- name: Unzip AWS Device Farm results
if: always()
run: unzip "Customer Artifacts.zip"
- name: Print AWS Device Farm run results
if: always()
run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
- name: Check if test failed, if so post the results and add the DeployBlocker label
id: checkIfRegressionDetected
run: |
if grep -q '🔴' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then
# Create an output to the GH action that the test failed:
echo "performanceRegressionDetected=true" >> "$GITHUB_OUTPUT"
gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
gh pr comment ${{ inputs.PR_NUMBER }} -F "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
else
echo "performanceRegressionDetected=false" >> "$GITHUB_OUTPUT"
echo '✅ no performance regression detected'
fi
env:
GITHUB_TOKEN: ${{ github.token }}
- name: 'Announce regression in Slack'
if: ${{ steps.checkIfRegressionDetected.outputs.performanceRegressionDetected == 'true' }}
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
channel: '#newdot-quality',
attachments: [{
color: 'danger',
text: `🔴 Performance regression detected in PR ${{ inputs.PR_NUMBER }}\nDetected in <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow.`,
}]
}
env:
GITHUB_TOKEN: ${{ github.token }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
cleanupDeltaRef:
needs: [prep, runTestsInAWS]
if: ${{ always() && needs.prep.outputs.IS_PR_MERGED != 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Delete temporary merge branch created for delta ref
run: git push -d origin ${{ needs.prep.outputs.DELTA_REF }}