mullvad · buggmagnet · Feb 27, 2024 · Feb 7, 2024
diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml
@@ -0,0 +1,80 @@
+---
+name: iOS end-to-end tests
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+  workflow_dispatch:
+jobs:
+  test:
+    if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch'
+    name: End to end tests
+    runs-on: [self-hosted, macOS, ios-test]
+    env:
+      IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+      TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+      TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+      HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER }}
+      NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure Rust
+        run: rustup target install aarch64-apple-ios aarch64-apple-ios-sim
+
+      - name: Configure Xcode project
+        run: |
+          for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
+          sed -i "" \
+            "/MULLVAD_IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
+            UITests.xcconfig
+          sed -i "" \
+            "/MULLVAD_TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
+            UITests.xcconfig
+          sed -i "" \
+            "/MULLVAD_HAS_TIME_ACCOUNT_NUMBER =/ s/= .*/= $HAS_TIME_ACCOUNT_NUMBER/" \
+            UITests.xcconfig
+          sed -i "" \
+            "/MULLVAD_NO_TIME_ACCOUNT_NUMBER =/ s/= .*/= $NO_TIME_ACCOUNT_NUMBER/" \
+            UITests.xcconfig
+        working-directory: ios/Configurations
+
+      - name: Run end-to-end-tests
+        run: |
+          set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
+            -project MullvadVPN.xcodeproj \
+            -scheme MullvadVPNUITests \
+            -testPlan MullvadVPNUITestsSmoke \
+            -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
+            test 2>&1 | xcbeautify --report junit --report-path test-report
+        working-directory: ios/
+
+      - name: Comment PR on test failure
+        if: failure() && github.event_name != 'workflow_dispatch'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const issue_number = context.issue.number;
+            const run_id = context.runId;
+            const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${run_id}`;
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issue_number,
+              body: `🚨 End to end tests failed. Please check the [failed workflow run](${run_url}).`
+            });
+
+      - name: Store test report artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report
+          path: ios/test-report/junit.xml
diff --git a/ios/MullvadVPN.xcodeproj/project.pbxproj b/ios/MullvadVPN.xcodeproj/project.pbxproj
@@ -1810,14 +1810,15 @@
 		7AF9BE8F2A39F26000DBFEDB /* Collection+Sorting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Collection+Sorting.swift"; sourceTree = "<group>"; };
 		7AF9BE942A40461100DBFEDB /* RelayFilterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterView.swift; sourceTree = "<group>"; };
 		7AF9BE962A41C71F00DBFEDB /* RelayFilterChipView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayFilterChipView.swift; sourceTree = "<group>"; };
+		85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITestsSmoke.xctestplan; sourceTree = "<group>"; };
 		850201DA2B503D7700EF8C96 /* RelayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayTests.swift; sourceTree = "<group>"; };
 		850201DC2B503D8C00EF8C96 /* SelectLocationPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectLocationPage.swift; sourceTree = "<group>"; };
 		850201DE2B5040A500EF8C96 /* TunnelControlPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TunnelControlPage.swift; sourceTree = "<group>"; };
 		850201E22B51A93C00EF8C96 /* SettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsPage.swift; sourceTree = "<group>"; };
 		8518F6372B60157E009EB113 /* LoggedInWithoutTimeUITestCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoggedInWithoutTimeUITestCase.swift; sourceTree = "<group>"; };
 		852969252B4D9C1F007EAD4C /* MullvadVPNUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MullvadVPNUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
 		852969272B4D9C1F007EAD4C /* AccountTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountTests.swift; sourceTree = "<group>"; };
-		852969302B4D9E70007EAD4C /* MullvadVPNUITests.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITests.xctestplan; sourceTree = "<group>"; };
+		852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = MullvadVPNUITestsAll.xctestplan; sourceTree = "<group>"; };
 		852969322B4E9232007EAD4C /* Page.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Page.swift; sourceTree = "<group>"; };
 		852969342B4E9270007EAD4C /* LoginPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoginPage.swift; sourceTree = "<group>"; };
 		852969372B4ED20E007EAD4C /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = "<group>"; };
@@ -3443,7 +3444,8 @@
 		7A83C3FC2A55B39500DFB83A /* TestPlans */ = {
 			isa = PBXGroup;
 			children = (
-				852969302B4D9E70007EAD4C /* MullvadVPNUITests.xctestplan */,
+				852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */,
+				85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */,
 				7A83C3FE2A55B72E00DFB83A /* MullvadVPNApp.xctestplan */,
 				7A83C4002A55B81A00DFB83A /* MullvadVPNCI.xctestplan */,
 				7A02D4EA2A9CEC7A00C19E31 /* MullvadVPNScreenshots.xctestplan */,

diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme
@@ -13,9 +13,12 @@
       shouldUseLaunchSchemeArgsEnv = "YES">
       <TestPlans>
          <TestPlanReference
-            reference = "container:TestPlans/MullvadVPNUITests.xctestplan"
+            reference = "container:TestPlans/MullvadVPNUITestsAll.xctestplan"
             default = "YES">
          </TestPlanReference>
+         <TestPlanReference
+            reference = "container:TestPlans/MullvadVPNUITestsSmoke.xctestplan">
+         </TestPlanReference>
       </TestPlans>
       <Testables>
          <TestableReference

diff --git a/ios/MullvadVPNUITests/README.md b/ios/MullvadVPNUITests/README.md
@@ -0,0 +1,34 @@
+# Integration tests
+
+## iOS device setup
+1. Make sure device is added to provisioning profiles
+2. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode
+3. Make sure device is configured in GitHub secrets(see *GitHub setup* below)
+
+## Set up of runner environment
+1. Install Xcode
+2. Sign in with Apple id in Xcode
+3. Download manual provisioning profiles in Xcode
+4. Install Xcode command line tools `xcode-select --install`
+5. Install yeetd
+ - `wget https://github.com/biscuitehh/yeetd/releases/download/1.0/yeetd-normal.pkg`
+ - `sudo installer -pkg yeetd-normal.pkg -target yeetd`
+6. Install Homebrew and dependencies
+  - `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
+  - `brew install xcbeautify wget swiftlint`
+7. Install Ruby
+  - `\curl -sSL https://get.rvm.io | bash`
+8. Install Rust and add iOS targets
+  - `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
+  - `rustup target install aarch64-apple-ios aarch64-apple-ios-sim`
+9. Install Go 1.19
+  - `brew install [email protected]`
+
+## GitHub runner setup
+1. Ask GitHub admin for new runner token and set it up according to the steps presented, pass `--labels ios-test` to `config.sh` when running it. By default it will also have the labels `self-hosted` and `macOS` which are required as well.
+2. Make sure GitHub actions secrets for the repository are correctly set up:
+  - `IOS_DEVICE_PIN_CODE` - Device passcode if the device require it, otherwise leave blank. Devices used with CI should not require passcode.
+  - `IOS_HAS_TIME_ACCOUNT_NUMBER` - Production server account without time left
+  - `IOS_NO_TIME_ACCOUNT_NUMBER` - Production server account with time added to it
+  - `IOS_TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`.
+  - `IOS_TEST_DEVICE_UDID` - the iOS device's UDID.
diff --git a/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift b/ios/MullvadVPNUITests/Test base classes/BaseUITestCase.swift
@@ -35,8 +35,10 @@ class BaseUITestCase: XCTestCase {
             alertAllowButton.tap()
         }
 
-        _ = springboard.buttons["1"].waitForExistence(timeout: Self.defaultTimeout)
-        springboard.typeText(iOSDevicePinCode)
+        if iOSDevicePinCode.isEmpty == false {
+            _ = springboard.buttons["1"].waitForExistence(timeout: Self.defaultTimeout)
+            springboard.typeText(iOSDevicePinCode)
+        }
     }
 
     // MARK: - Setup & teardown

diff --git a/ios/TestPlans/MullvadVPNUITests.xctestplan → ...TestPlans/MullvadVPNUITestsAll.xctestplan b/ios/TestPlans/MullvadVPNUITests.xctestplan → ...TestPlans/MullvadVPNUITestsAll.xctestplan
@@ -17,6 +17,10 @@
   },
   "testTargets" : [
     {
+      "selectedTests" : [
+        "AccountTests\/testLogin()",
+        "AccountTests\/testLoginWithIncorrectAccountNumber()"
+      ],
       "target" : {
         "containerPath" : "container:MullvadVPN.xcodeproj",
         "identifier" : "852969242B4D9C1F007EAD4C",

diff --git a/ios/TestPlans/MullvadVPNUITestsSmoke.xctestplan b/ios/TestPlans/MullvadVPNUITestsSmoke.xctestplan
@@ -0,0 +1,28 @@
+{
+  "configurations" : [
+    {
+      "id" : "3BCBAC52-59BD-45E8-BC9D-1AF65D1B15B7",
+      "name" : "Configuration 1",
+      "options" : {
+
+      }
+    }
+  ],
+  "defaultOptions" : {
+
+  },
+  "testTargets" : [
+    {
+      "selectedTests" : [
+        "AccountTests\/testLogin()",
+        "AccountTests\/testLoginWithIncorrectAccountNumber()"
+      ],
+      "target" : {
+        "containerPath" : "container:MullvadVPN.xcodeproj",
+        "identifier" : "852969242B4D9C1F007EAD4C",
+        "name" : "MullvadVPNUITests"
+      }
+    }
+  ],
+  "version" : 1
+}