From ccd64d7522fb0f00aef44c8bc3fa5faf6e34d38b Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Tue, 19 Nov 2024 15:45:51 -0800 Subject: [PATCH] replace App Center with BrowserStack --- ...appcenter.yml => android-browserstack.yml} | 68 ++++----- .github/workflows/android-perf.yml | 38 +++-- .github/workflows/ios-appcenter.yml | 71 ---------- .github/workflows/ios-browserstack.yml | 79 +++++++++++ .github/workflows/ios-demos.yml | 3 - .github/workflows/ios-perf.yml | 55 ++++--- .../EagleTestApp/eagle-test-app/build.gradle | 1 - .../ai/picovoice/eagle/testapp/BaseTest.java | 14 -- .../eagle/testapp/IntegrationTest.java | 11 -- script/automation/browserstack.py | 134 ++++++++++++++++++ 10 files changed, 297 insertions(+), 177 deletions(-) rename .github/workflows/{android-appcenter.yml => android-browserstack.yml} (62%) delete mode 100644 .github/workflows/ios-appcenter.yml create mode 100644 .github/workflows/ios-browserstack.yml create mode 100644 script/automation/browserstack.py diff --git a/.github/workflows/android-appcenter.yml b/.github/workflows/android-browserstack.yml similarity index 62% rename from .github/workflows/android-appcenter.yml rename to .github/workflows/android-browserstack.yml index 9765a260..41ef3cfb 100644 --- a/.github/workflows/android-appcenter.yml +++ b/.github/workflows/android-browserstack.yml @@ -1,4 +1,4 @@ -name: Android AppCenter Tests +name: Android BrowserStack Tests on: workflow_dispatch: @@ -6,13 +6,13 @@ on: branches: [ main ] paths: - 'binding/android/EagleTestApp/**' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' - 'resources/audio_samples/**' pull_request: branches: [ main, 'v[0-9]+.[0-9]+' ] paths: - 'binding/android/EagleTestApp/**' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' - 'resources/audio_samples/**' defaults: @@ -21,17 +21,18 @@ defaults: jobs: build: - name: Run Android Tests on AppCenter + name: Run Android Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Setup Node.js environment - uses: actions/setup-node@v3 - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + - name: Installing Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -61,30 +62,29 @@ jobs: - name: Build androidTest run: ./gradlew assembleDebugAndroidTest - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Eagle-Android" - --devices "Picovoice/android-min-max" - --app-path eagle-test-app/build/outputs/apk/debug/eagle-test-app-debug.apk - --test-series "eagle-android" - --locale "en_US" - --build-dir eagle-test-app/build/outputs/apk/androidTest/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Eagle-Android" + --devices "android-min-max" + --app_path "eagle-test-app/build/outputs/apk/debug/eagle-test-app-debug.apk" + --test_path "eagle-test-app/build/outputs/apk/androidTest/debug/eagle-test-app-debug-androidTest.apk" build-integ: - name: Run Android Integration Tests on AppCenter + name: Run Android Integration Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -117,12 +117,12 @@ jobs: - name: Build androidTest run: ./gradlew assembleReleaseAndroidTest -DtestBuildType=integ - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Eagle-Android" - --devices "Picovoice/android-min-max" - --app-path eagle-test-app/build/outputs/apk/release/eagle-test-app-release.apk - --test-series "eagle-android" - --locale "en_US" - --build-dir eagle-test-app/build/outputs/apk/androidTest/release + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Eagle-Android-Integration" + --devices "android-min-max" + --app_path "eagle-test-app/build/outputs/apk/release/eagle-test-app-release.apk" + --test_path "eagle-test-app/build/outputs/apk/androidTest/release/eagle-test-app-release-androidTest.apk" diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 6ebb9bef..9acc7fbc 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -21,28 +21,26 @@ defaults: jobs: build: - name: Run Android Speed Tests on AppCenter + name: Run Android Speed Tests on BrowserStack runs-on: ubuntu-latest strategy: matrix: - device: [single-android, 32bit-android] + device: [ android-perf ] include: - - device: single-android + - device: android-perf enrollPerformanceThresholdSec: 0.6 procPerformanceThresholdSec: 0.6 - - device: 32bit-android - enrollPerformanceThresholdSec: 5.5 - procPerformanceThresholdSec: 5.5 steps: - uses: actions/checkout@v3 - - name: Setup Node.js environment - uses: actions/setup-node@v3 - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + - name: Installing Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -84,12 +82,12 @@ jobs: - name: Build androidTest run: ./gradlew assembleDebugAndroidTest - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Eagle-Android" - --devices "Picovoice/${{ matrix.device }}" - --app-path eagle-test-app/build/outputs/apk/debug/eagle-test-app-debug.apk - --test-series "eagle-android" - --locale "en_US" - --build-dir eagle-test-app/build/outputs/apk/androidTest/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Eagle-Android-Performance" + --devices "${{ matrix.device }}" + --app_path "eagle-test-app/build/outputs/apk/debug/eagle-test-app-debug.apk" + --test_path "eagle-test-app/build/outputs/apk/androidTest/debug/eagle-test-app-debug-androidTest.apk" diff --git a/.github/workflows/ios-appcenter.yml b/.github/workflows/ios-appcenter.yml deleted file mode 100644 index 3f0734ac..00000000 --- a/.github/workflows/ios-appcenter.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: iOS AppCenter Tests - -on: - workflow_dispatch: - push: - branches: [ main ] - paths: - - '.github/workflows/ios-appcenter.yml' - - 'binding/ios/EagleAppTest/**' - - 'resources/audio_samples/**' - pull_request: - branches: [ main, 'v[0-9]+.[0-9]+' ] - paths: - - '.github/workflows/ios-appcenter.yml' - - 'binding/ios/EagleAppTest/**' - - 'resources/audio_samples/**' - -defaults: - run: - working-directory: binding/ios/EagleAppTest - -jobs: - build: - name: Run iOS Tests on AppCenter - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Node.js environment - uses: actions/setup-node@v3 - - - name: Install Cocoapods - run: gem install cocoapods - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - - name: Make build dir - run: mkdir ddp - - - name: Install resource script dependency - run: | - brew update - brew install convmv - - - name: Run Cocoapods - run: pod install - - - name: Inject AppID - run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' - EagleAppTestUITests/BaseTest.swift - - - name: XCode Build - run: xcrun xcodebuild build-for-testing - -configuration Debug - -workspace EagleAppTest.xcworkspace - -sdk iphoneos - -scheme EagleAppTest - -derivedDataPath ddp - CODE_SIGNING_ALLOWED=NO - - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Eagle-iOS" - --devices "Picovoice/ios-min-max" - --test-series "eagle-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos diff --git a/.github/workflows/ios-browserstack.yml b/.github/workflows/ios-browserstack.yml new file mode 100644 index 00000000..750380e3 --- /dev/null +++ b/.github/workflows/ios-browserstack.yml @@ -0,0 +1,79 @@ +name: iOS BrowserStack Tests + +on: + workflow_dispatch: + push: + branches: [ main ] + paths: + - '.github/workflows/ios-browserstack.yml' + - 'binding/ios/EagleAppTest/**' + - 'resources/audio_samples/**' + pull_request: + branches: [ main, 'v[0-9]+.[0-9]+' ] + paths: + - '.github/workflows/ios-browserstack.yml' + - 'binding/ios/EagleAppTest/**' + - 'resources/audio_samples/**' + +defaults: + run: + working-directory: binding/ios/EagleAppTest + +jobs: + build: + name: Run iOS Tests on BrowserStack + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Installing Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: + pip3 install requests + + - name: Install Cocoapods + run: gem install cocoapods + + - name: Make build dir + run: mkdir ddp + + - name: Run Cocoapods + run: pod install + + - name: Inject AccessKey + run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' + EagleAppTestUITests/BaseTest.swift + + - name: XCode Build + run: xcrun xcodebuild build-for-testing + -configuration Debug + -workspace EagleAppTest.xcworkspace + -sdk iphoneos + -scheme EagleAppTest + -derivedDataPath ddp + CODE_SIGNING_ALLOWED=NO + + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r EagleAppTest.app Payload && + zip --symlinks -r EagleAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r EagleAppTestUITests.zip EagleAppTestUITests-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Eagle-iOS" + --devices "ios-min-max" + --app_path "ddp/Build/Products/Debug-iphoneos/EagleAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/EagleAppTestUITests.zip" diff --git a/.github/workflows/ios-demos.yml b/.github/workflows/ios-demos.yml index 8a111f4f..580a0740 100644 --- a/.github/workflows/ios-demos.yml +++ b/.github/workflows/ios-demos.yml @@ -33,9 +33,6 @@ jobs: - name: Install Cocoapods run: gem install cocoapods - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - name: Make build dir run: mkdir ddp diff --git a/.github/workflows/ios-perf.yml b/.github/workflows/ios-perf.yml index 764ed48a..559516d4 100644 --- a/.github/workflows/ios-perf.yml +++ b/.github/workflows/ios-perf.yml @@ -21,12 +21,12 @@ defaults: jobs: build: - name: Run iOS Tests on AppCenter + name: Run iOS Tests on BrowserStack runs-on: macos-latest strategy: matrix: - device: [ios-perf] + device: [ ios-perf ] include: - device: ios-perf enrollPerformanceThresholdSec: 0.5 @@ -36,27 +36,22 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Setup Node.js environment - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 + python-version: '3.10' + - run: + pip3 install requests - name: Install Cocoapods run: gem install cocoapods - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - name: Make build dir run: mkdir ddp - - name: Install resource script dependency - run: | - brew update - brew install convmv - - name: Run Cocoapods run: pod install - - name: Inject AppID + - name: Inject AccessKey run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' PerformanceTest/PerformanceTest.swift @@ -65,11 +60,13 @@ jobs: PerformanceTest/PerformanceTest.swift - name: Inject Performance Threshold - run: sed -i '.bak' 's:{ENROLL_PERFORMANCE_THRESHOLD_SEC}:${{ matrix.enrollPerformanceThresholdSec }}:' + run: sed -i '.bak' + '1,/{ENROLL_PERFORMANCE_THRESHOLD_SEC}/s/{ENROLL_PERFORMANCE_THRESHOLD_SEC}/${{ matrix.enrollPerformanceThresholdSec }}/' PerformanceTest/PerformanceTest.swift - name: Inject Performance Threshold - run: sed -i '.bak' 's:{PROC_PERFORMANCE_THRESHOLD_SEC}:${{ matrix.procPerformanceThresholdSec }}:' + run: sed -i '.bak' + '1,/{PROC_PERFORMANCE_THRESHOLD_SEC}/s/{PROC_PERFORMANCE_THRESHOLD_SEC}/${{ matrix.procPerformanceThresholdSec }}/' PerformanceTest/PerformanceTest.swift - name: XCode Build @@ -81,11 +78,23 @@ jobs: -derivedDataPath ddp CODE_SIGNING_ALLOWED=NO - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Eagle-iOS" - --devices "Picovoice/${{ matrix.device }}" - --test-series "eagle-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r EagleAppTest.app Payload && + zip --symlinks -r EagleAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r PerformanceTest.zip PerformanceTest-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Eagle-iOS-Performance" + --devices "${{ matrix.device }}" + --app_path "ddp/Build/Products/Debug-iphoneos/EagleAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/PerformanceTest.zip" diff --git a/binding/android/EagleTestApp/eagle-test-app/build.gradle b/binding/android/EagleTestApp/eagle-test-app/build.gradle index 1c0b0ef8..9ec7b4b9 100644 --- a/binding/android/EagleTestApp/eagle-test-app/build.gradle +++ b/binding/android/EagleTestApp/eagle-test-app/build.gradle @@ -123,7 +123,6 @@ dependencies { androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { exclude group: 'com.android.support', module: 'support-annotations' }) - androidTestImplementation('com.microsoft.appcenter:espresso-test-extension:1.4') androidTestImplementation('androidx.test.espresso:espresso-intents:3.5.1') } diff --git a/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/BaseTest.java b/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/BaseTest.java index d0d1f22f..a905e7a5 100644 --- a/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/BaseTest.java +++ b/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/BaseTest.java @@ -17,12 +17,7 @@ import androidx.test.platform.app.InstrumentationRegistry; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -45,10 +40,6 @@ public class BaseTest { protected final String testPath = "audio_samples/speaker_1_test_utt.wav"; protected final String imposterPath = "audio_samples/speaker_2_test_utt.wav"; - - @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); - Context testContext; Context appContext; AssetManager assetManager; @@ -57,11 +48,6 @@ public class BaseTest { String accessKey; - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Before public void Setup() throws IOException { testContext = InstrumentationRegistry.getInstrumentation().getContext(); diff --git a/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/IntegrationTest.java b/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/IntegrationTest.java index 680461c3..f1684890 100644 --- a/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/IntegrationTest.java +++ b/binding/android/EagleTestApp/eagle-test-app/src/androidTest/java/ai/picovoice/eagle/testapp/IntegrationTest.java @@ -16,9 +16,6 @@ import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - import org.hamcrest.Matcher; import org.junit.After; import org.junit.Before; @@ -74,9 +71,6 @@ public void perform(UiController uiController, View view) { @RunWith(AndroidJUnit4.class) public class IntegrationTest { - @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); - @Rule public ActivityScenarioRule activityScenarioRule = new ActivityScenarioRule<>(MainActivity.class); @@ -91,11 +85,6 @@ public void intentsTeardown() { Intents.release(); } - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Test public void testEagle() { onView(withId(R.id.testButton)).perform(click()); diff --git a/script/automation/browserstack.py b/script/automation/browserstack.py new file mode 100644 index 00000000..bfdf1f49 --- /dev/null +++ b/script/automation/browserstack.py @@ -0,0 +1,134 @@ +import argparse +import requests +import time + +APP_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/app' +TEST_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/test-suite' +BUILD_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/build' +STATUS_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/builds/{}' + +devices_dict = { + 'android-min-max': [ + 'Samsung Galaxy S8-7.0', + 'Samsung Galaxy M52-11.0', + 'Google Pixel 9-15.0' + ], + 'android-perf': [ + 'Google Pixel 6 Pro-15.0' + ], + 'ios-min-max': [ + 'iPhone SE 2020-13', + 'iPhone 14 Pro-16', + 'iPhone 14-18' + ], + 'ios-perf': [ + 'iPhone 13-18', + ] +} + + +def main(args: argparse.Namespace) -> None: + app_files = { + 'file': open(args.app_path, 'rb') + } + + app_response = requests.post( + APP_URI.format(args.type), + files=app_files, + auth=(args.username, args.access_key) + ) + app_response_json = app_response.json() + + if not app_response.ok: + print('App Upload Failed', app_response_json) + exit(1) + + test_files = { + 'file': open(args.test_path, 'rb') + } + test_response = requests.post( + TEST_URI.format(args.type), + files=test_files, + auth=(args.username, args.access_key) + ) + test_response_json = test_response.json() + + if not test_response.ok: + print('Test Upload Failed', test_response_json) + exit(1) + + build_headers = { + 'Content-Type': 'application/json' + } + build_data = { + 'app': app_response_json['app_url'], + 'testSuite': test_response_json['test_suite_url'], + 'project': args.project_name, + 'devices': devices_dict[args.devices] + } + + while True: + build_response = requests.post( + BUILD_URI.format(args.type), + headers=build_headers, + json=build_data, + auth=(args.username, args.access_key) + ) + if (build_response is not None and 'message' in build_response.json() and '[BROWSERSTACK_ALL_PARALLELS_IN_USE]' + in build_response.json()['message']): + print('Parallel threads limit reached. Waiting...', flush=True) + time.sleep(60) + else: + break + + if build_response is None: + print('Build Failed') + exit(1) + + build_response_json = build_response.json() + + if not build_response.ok: + print('Build Failed', build_response.json()) + exit(1) + + if build_response_json['message'] != 'Success': + print('Build Unsuccessful') + exit(1) + + print('View build results at https://app-automate.browserstack.com/dashboard/v2/builds/{}' + .format(build_response_json['build_id'])) + + while True: + time.sleep(60) + status_response = requests.get( + STATUS_URI.format(args.type, build_response_json['build_id']), + auth=(args.username, args.access_key) + ) + status_response_json = status_response.json() + status = status_response_json['status'] + + if not status_response.ok: + print('Status Request Failed', status_response_json) + exit(1) + + if status != 'queued' and status != 'running': + break + + print('Status:', status) + if status != 'passed': + exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--type', choices=['espresso', 'xcuitest'], required=True) + parser.add_argument('--username', required=True) + parser.add_argument('--access_key', required=True) + + parser.add_argument('--project_name', required=True) + parser.add_argument('--devices', choices=devices_dict.keys(), required=True) + parser.add_argument('--app_path', required=True) + parser.add_argument('--test_path', required=True) + args = parser.parse_args() + + main(args)