From c3c9d36f27affba415e72585c27bb8827aee6286 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 7 Feb 2024 17:40:35 +0100 Subject: [PATCH] GitHub actions workflow for iOS end to end tests --- .github/workflows/ios-end-to-end-tests.yml | 80 +++++++++++++++++++ ios/MullvadVPN.xcodeproj/project.pbxproj | 6 +- .../xcschemes/MullvadVPNUITests.xcscheme | 5 +- ios/MullvadVPNUITests/README.md | 34 ++++++++ .../Test base classes/BaseUITestCase.swift | 6 +- ...stplan => MullvadVPNUITestsAll.xctestplan} | 4 + .../MullvadVPNUITestsSmoke.xctestplan | 28 +++++++ 7 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ios-end-to-end-tests.yml create mode 100644 ios/MullvadVPNUITests/README.md rename ios/TestPlans/{MullvadVPNUITests.xctestplan => MullvadVPNUITestsAll.xctestplan} (81%) create mode 100644 ios/TestPlans/MullvadVPNUITestsSmoke.xctestplan diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml new file mode 100644 index 000000000000..28da6da122c0 --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -0,0 +1,80 @@ +--- +name: iOS end-to-end tests +permissions: + contents: read + issues: write + pull-requests: write +on: + pull_request: + types: + - closed + branches: + - main + workflow_dispatch: +jobs: + test: + if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' + name: End to end tests + runs-on: [self-hosted, macOS, ios-test] + env: + IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }} + TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} + TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }} + HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER }} + NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configure Rust + run: rustup target install aarch64-apple-ios aarch64-apple-ios-sim + + - name: Configure Xcode project + run: | + for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done + sed -i "" \ + "/MULLVAD_IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \ + UITests.xcconfig + sed -i "" \ + "/MULLVAD_TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ + UITests.xcconfig + sed -i "" \ + "/MULLVAD_HAS_TIME_ACCOUNT_NUMBER =/ s/= .*/= $HAS_TIME_ACCOUNT_NUMBER/" \ + UITests.xcconfig + sed -i "" \ + "/MULLVAD_NO_TIME_ACCOUNT_NUMBER =/ s/= .*/= $NO_TIME_ACCOUNT_NUMBER/" \ + UITests.xcconfig + working-directory: ios/Configurations + + - name: Run end-to-end-tests + run: | + set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ + -project MullvadVPN.xcodeproj \ + -scheme MullvadVPNUITests \ + -testPlan MullvadVPNUITestsSmoke \ + -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ + test 2>&1 | xcbeautify --report junit --report-path test-report + working-directory: ios/ + + - name: Comment PR on test failure + if: failure() && github.event_name != 'workflow_dispatch' + uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const issue_number = context.issue.number; + const run_id = context.runId; + const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${run_id}`; + github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: `🚨 End to end tests failed. Please check the [failed workflow run](${run_url}).` + }); + + - name: Store test report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-report + path: ios/test-report/junit.xml diff --git a/ios/MullvadVPN.xcodeproj/project.pbxproj b/ios/MullvadVPN.xcodeproj/project.pbxproj index c648222b8f8d..a0943f1f1ab6 100644 --- a/ios/MullvadVPN.xcodeproj/project.pbxproj +++ b/ios/MullvadVPN.xcodeproj/project.pbxproj @@ -1810,6 +1810,7 @@ 7AF9BE8F2A39F26000DBFEDB /* Collection+Sorting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Collection+Sorting.swift"; sourceTree = ""; }; 7AF9BE942A40461100DBFEDB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = ""; }; 7AF9BE962A41C71F00DBFEDB /* RelayFilterChipView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterChipView.swift; sourceTree = ""; }; + 85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITestsSmoke.xctestplan; sourceTree = ""; }; 850201DA2B503D7700EF8C96 /* RelayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayTests.swift; sourceTree = ""; }; 850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectLocationPage.swift; sourceTree = ""; }; 850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TunnelControlPage.swift; sourceTree = ""; }; @@ -1817,7 +1818,7 @@ 8518F6372B60157E009EB113 /* LoggedInWithoutTimeUITestCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoggedInWithoutTimeUITestCase.swift; sourceTree = ""; }; 852969252B4D9C1F007EAD4C /* MullvadVPNUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MullvadVPNUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 852969272B4D9C1F007EAD4C /* AccountTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountTests.swift; sourceTree = ""; }; - 852969302B4D9E70007EAD4C /* MullvadVPNUITests.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITests.xctestplan; sourceTree = ""; }; + 852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITestsAll.xctestplan; sourceTree = ""; }; 852969322B4E9232007EAD4C /* Page.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Page.swift; sourceTree = ""; }; 852969342B4E9270007EAD4C /* LoginPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoginPage.swift; sourceTree = ""; }; 852969372B4ED20E007EAD4C /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; @@ -3443,7 +3444,8 @@ 7A83C3FC2A55B39500DFB83A /* TestPlans */ = { isa = PBXGroup; children = ( - 852969302B4D9E70007EAD4C /* MullvadVPNUITests.xctestplan */, + 852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */, + 85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */, 7A83C3FE2A55B72E00DFB83A /* MullvadVPNApp.xctestplan */, 7A83C4002A55B81A00DFB83A /* MullvadVPNCI.xctestplan */, 7A02D4EA2A9CEC7A00C19E31 /* MullvadVPNScreenshots.xctestplan */, diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index 7c65faea3b98..7603c9373776 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -13,9 +13,12 @@ shouldUseLaunchSchemeArgsEnv = "YES"> + +