test: fix tests for lower API levels

.github/workflows/ci.yml

@@ -102,7 +102,8 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        api-level: [34]
+        api-level: [29, 30, 31, 33, 35]
+      fail-fast: false # make sure all tests finish, even if some fail
 
     steps:
       - uses: actions/checkout@v4
@@ -116,33 +117,30 @@ jobs:
           cache-encryption-key: ${{ secrets.GRADLE_CACHE_ENCRYPTION_KEY }}
           cache-read-only: true
 
-      # API 30+ emulators only have x86_64 system images.
-      - name: Get AVD info
-        uses: ./.github/actions/get-avd-info
-        id: avd-info
-        with:
-          api-level: ${{ matrix.api-level }}
-
+      - name: Accept licenses
+        run:  yes | "$ANDROID_HOME"/cmdline-tools/latest/bin/sdkmanager --licenses
+
       - name: Enable KVM group perms
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
           sudo udevadm control --reload-rules
           sudo udevadm trigger --name-match=kvm
-    
-      - name: Instrumentation tests
-        uses: reactivecircus/[email protected]
+
+      - name: Cache AVD
+        uses: actions/cache@v4
         with:
-          api-level: ${{ matrix.api-level }}
-          arch: ${{ steps.avd-info.outputs.arch }}
-          target: ${{ steps.avd-info.outputs.target }}
-          script: ./gradlew --build-cache --configuration-cache --configuration-cache-problems=warn app:connectedDebugAndroidTest
+          path: ~/.config/.android/avd
+          key: avd-${{ hashFiles('app/build.gradle.kts') }}   # gradle-managed devices are defined there
+
+      - name: Run device tests
+        run: ./gradlew --build-cache --configuration-cache --configuration-cache-problems=warn app:api${{ matrix.api-level }}DebugAndroidTest --stacktrace
 
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: instrumentation-test-results-${{ matrix.api-level }}
-          path: app/build/reports/androidTests/connected/debug
+          path: app/build/reports/androidTests/managedDevice/debug/api${{ matrix.api-level }}
 
   final-check:
     name: CI checks passed

app/build.gradle.kts

@@ -87,10 +87,51 @@ android {
         targetCompatibility = javaVersion
     }
 
-    // needed for mockk
     testOptions {
         animationsDisabled = true
 
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
+
+        managedDevices {
+            localDevices {
+                create("api29") {
+                    device = "Pixel 6a"
+                    apiLevel = 29
+                    systemImageSource = "aosp"
+                }
+                create("api30") {
+                    device = "Pixel 6a"
+                    apiLevel = 30
+                    systemImageSource = "aosp-atd"
+                }
+                create("api31") {
+                    device = "Pixel 6a"
+                    apiLevel = 31
+                    systemImageSource = "aosp-atd"
+                }
+                create("api33") {
+                    device = "Pixel 6a"
+                    apiLevel = 33
+                    systemImageSource = "aosp-atd"
+                }
+                create("api35") {
+                    device = "Pixel 6a"
+                    apiLevel = 35
+                    systemImageSource = "aosp-atd"
+                }
+            }
+            groups {
+                create("all") {
+                    targetDevices.add(devices["api29"])
+                    targetDevices.add(devices["api30"])
+                    targetDevices.add(devices["api31"])
+                    targetDevices.add(devices["api33"])
+                    targetDevices.add(devices["api35"])
+                }
+            }
+        }
+
+        // needed for mockk
         packaging {
             jniLibs { useLegacyPackaging = true }
         }
@@ -220,6 +261,26 @@ tasks.register<FixLicenseTask>("fixLicense") {
     description = "Fixes the license header in all staged files."
 }
 
+val embedScreenshotsTask = tasks.register<EmbedScreenshotsTask>("embedScreenshots") {
+    dependsOn("createDebugApkListingFileRedirect")
+    dependsOn("createDebugAndroidTestApkListingFileRedirect")
+
+    buildIntermediatesDir.set(
+        project.layout.projectDirectory.dir("build/intermediates")
+    )
+    managedDeviceReportDirectory.set(
+        project.layout.projectDirectory.dir("build/reports/androidTests/managedDevice/debug")
+    )
+}
+
+tasks.whenTaskAdded {
+    // Do not attempt to embed screenshots for API level < 29 since
+    // additional_test_output is not supported.
+    if (name.matches(regex = Regex("api[3-9][0-9]DebugAndroidTest"))) {
+        finalizedBy(embedScreenshotsTask)
+    }
+}
+
 dependencies {
     detektPlugins(libs.detekt.formatting)
 
@@ -318,6 +379,7 @@ dependencies {
     debugImplementation(libs.androidx.compose.ui.test.manifest)
 
     // Instrumentation tests
+    androidTestUtil(libs.androidx.test.orchestrator)
     androidTestImplementation(libs.hilt.android.testing)
     kspAndroidTest(libs.hilt.android.compiler)
     androidTestImplementation(libs.junit)
@@ -405,3 +467,81 @@ abstract class SetupMusikusTask @Inject constructor(
         println("Name stored for copyright header: $name\n")
     }
 }
+
+abstract class EmbedScreenshotsTask : DefaultTask() {
+
+    @InputDirectory
+    val buildIntermediatesDir = project.objects.directoryProperty()
+
+    @OutputDirectory
+    val managedDeviceReportDirectory = project.objects.directoryProperty()
+
+    @get:Inject abstract val fs: FileSystemOperations
+
+    init {
+        group = "verification"
+        description = "Embeds screenshots into JUnit test reports."
+    }
+
+    @TaskAction
+    fun embedScreenshots() {
+        println("Embedding screenshots into JUnit reports...")
+
+        val buildIntermediatesDirFile = buildIntermediatesDir.asFile.get()
+        val managedDeviceReportDirectoryFile = managedDeviceReportDirectory.asFile.get()
+
+        val additionalTestOutputDirFile = File(
+            "$buildIntermediatesDirFile/managed_device_android_test_additional_output/debugAndroidTest"
+        )
+
+        val deviceDirectory = additionalTestOutputDirFile.listFiles()?.firstOrNull()
+
+        if (deviceDirectory == null) {
+            println("No device directory found in '$additionalTestOutputDirFile'")
+            return
+        }
+        val deviceName = deviceDirectory.name.replace("DebugAndroidTest", "")
+
+        println("Processing screenshots for device '$deviceName'...")
+
+        val screenshotDirectory = File(managedDeviceReportDirectoryFile, "$deviceName/screenshots")
+
+        println("Copying screenshots to '$screenshotDirectory'...")
+
+        fs.copy {
+            from(deviceDirectory)
+            into(screenshotDirectory)
+        }
+
+        screenshotDirectory.listFiles()?.forEach { failedTestClassDirectory ->
+            println("Processing screenshots for test class '${failedTestClassDirectory.name}'...")
+
+            val failedTestClassName = failedTestClassDirectory.name
+
+            failedTestClassDirectory.listFiles()?.forEach listFiles@{ failedTestFile ->
+                println("Embedding screenshot for test '$failedTestFile'...")
+                val failedTestName = failedTestFile.name
+                val failedTestNameWithoutExtension = failedTestName.substringBeforeLast('.')
+                val failedTestClassJunitReportFile = File(
+                    managedDeviceReportDirectoryFile,
+                    "$deviceName/$failedTestClassName.html"
+                )
+
+                if (!failedTestClassJunitReportFile.exists()) {
+                    println("Could not find JUnit report file for test class '$failedTestClassJunitReportFile'")
+                    return@listFiles
+                }
+
+                var failedTestJunitReportContent = failedTestClassJunitReportFile.readText()
+
+                val patternToFind = "<h3 class=\"failures\">$failedTestNameWithoutExtension</h3>"
+                val patternToReplace =
+                    "$patternToFind <img src=\"screenshots/$failedTestClassName/$failedTestName\" width=\"360\" /></br>"
+
+                failedTestJunitReportContent = failedTestJunitReportContent.replace(patternToFind, patternToReplace)
+
+                failedTestClassJunitReportFile.writeText(failedTestJunitReportContent)
+            }
+        }
+    }
+}

app/src/androidTest/java/app/ComposeRule.kt

@@ -0,0 +1,37 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) 2025 Matthias Emde
+ */
+
+package app
+
+import androidx.compose.ui.test.SemanticsNodeInteraction
+import androidx.compose.ui.test.getBoundsInRoot
+
+/**
+ * Asserts that the given SemanticsNodeInteractions are vertically ordered on the screen.
+ *
+ * This function iterates through the provided nodes and verifies that each node is
+ * positioned above the subsequent node in the list. It checks if the bottom bound
+ * of the current node is less than or equal to the top bound of the next node.
+ *
+ * If any two consecutive nodes violate this vertical order, an assertion error is thrown.
+ *
+ * @param nodes The SemanticsNodeInteractions to be checked for vertical order.
+ *
+ * @throws AssertionError If any two consecutive nodes are not vertically ordered as expected.
+ */
+fun assertNodesInVerticalOrder(vararg nodes: SemanticsNodeInteraction) {
+    for (i in 0 until nodes.size - 1) {
+        val currentBounds = nodes[i].getBoundsInRoot()
+        val nextBounds = nodes[i + 1].getBoundsInRoot()
+
+        check(currentBounds.bottom <= nextBounds.top) {
+            "Expected node ${i + 1} to be above node ${i + 2}, but was not. " +
+                "Bounds: current = $currentBounds, next = $nextBounds"
+        }
+    }
+}

app/src/androidTest/java/app/ScreenshotRule.kt

@@ -0,0 +1,65 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) 2024 Matthias Emde
+ */
+
+package app
+
+import android.graphics.Bitmap
+import androidx.compose.ui.graphics.asAndroidBitmap
+import androidx.compose.ui.test.captureToImage
+import androidx.compose.ui.test.junit4.ComposeTestRule
+import androidx.compose.ui.test.onRoot
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.rules.TestWatcher
+import org.junit.runner.Description
+import java.io.File
+
+// Store screenshots in "/sdcard/Android/media/app.musikus/additional_test_output"
+// This way, the files will get automatically synced to app/build/outputs/managed_device_android_test_additional_output
+// before the emulator gets shut down.
+// Source: https://stackoverflow.com/questions/74069309/copy-data-from-an-android-emulator-that-is-run-by-gradle-managed-devices
+
+class ScreenshotRule(
+    private val composeTestRule: ComposeTestRule,
+) : TestWatcher() {
+
+    var outputDir: File
+
+    init {
+        @Suppress("Deprecation")
+        outputDir = File(
+            InstrumentationRegistry.getInstrumentation().targetContext.externalMediaDirs.first(),
+            "additional_test_output"
+        )
+
+        // Ensure the directory exists
+        if (!outputDir.exists()) {
+            outputDir.mkdirs()
+        }
+    }
+
+    override fun failed(e: Throwable?, description: Description) {
+        val testClassDir = File(
+            outputDir,
+            description.className
+        )
+
+        if (!testClassDir.exists()) {
+            testClassDir.mkdirs()
+        }
+
+        val screenshotName = "${description.methodName}.png"
+        val screenshotFile = File(testClassDir, screenshotName)
+
+        // Capture the screenshot and save it
+        composeTestRule.onRoot().captureToImage().asAndroidBitmap().apply {
+            screenshotFile.outputStream().use { outputStream ->
+                compress(Bitmap.CompressFormat.PNG, 100, outputStream)
+            }
+        }
+    }
+}

app/src/androidTest/java/app/musikus/activesession/presentation/ActiveSessionScreenTest.kt

@@ -21,6 +21,8 @@ import androidx.compose.ui.test.onNodeWithText
 import androidx.compose.ui.test.performClick
 import androidx.compose.ui.test.performTextInput
 import androidx.navigation.NavHostController
+import androidx.test.filters.SdkSuppress
+import app.ScreenshotRule
 import app.musikus.core.data.Nullable
 import app.musikus.core.data.SectionWithLibraryItem
 import app.musikus.core.data.SessionWithSectionsWithLibraryItems
@@ -65,6 +67,9 @@ class ActiveSessionScreenTest {
     @get:Rule(order = 1)
     val composeRule = createAndroidComposeRule<MainActivity>()
 
+    @get:Rule(order = 2)
+    val screenshotRule = ScreenshotRule(composeRule)
+
     lateinit var navController: NavHostController
     lateinit var mainViewModel: MainViewModel
 
@@ -212,6 +217,7 @@ class ActiveSessionScreenTest {
     }
 
 //    @Test
+//    @SdkSuppress(minSdkVersion = 30)
 //    fun deleteAndRedoSection() = runTest {
 //        // Start session
 //        composeRule.onNodeWithContentDescription("Start practicing").performClick()
@@ -275,6 +281,7 @@ class ActiveSessionScreenTest {
      * Section: 00000000-0000-0000-0000-000000000007
      */
     @Test
+    @SdkSuppress(minSdkVersion = 29) // somehow crashes on API < 29
     fun finishSession() = runTest {
         // Start session
         composeRule.onNodeWithContentDescription("Start practicing").performClick()