tests now search the envelope among all the ones sent to the mock rel…

…ay server and make assertion on them, instead of relying on the order the envelopes were created (#2422)

* ui tests now search the envelope among all the ones sent to the mock relay server and make assertion on them, instead of relying on the order the envelopes were created
* added androidx test orchestrator to saucelabs configuration with "clearPackageData" option enabled
* updated androidx orchestrator dependency

.sauce/sentry-uitest-android-benchmark-lite.yml

@@ -19,6 +19,9 @@ espresso:
 suites:
 
   - name: "Android 11 (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11)
 

.sauce/sentry-uitest-android-benchmark.yml

@@ -20,17 +20,26 @@ suites:
 
   # Devices are chosen so that there is a high-end and a low-end device for each api level
   - name: "Android 12 (api 31)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) - high end
       - id: Google_Pixel_6a_real_us # Google Pixel 6a - api 31 (12) - low end
 
   - name: "Android 11 (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11) - high end
       - id: Google_Pixel_4_real_us # Google Pixel 4 - api 30 (11) - mid end
       - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) - low end
 
   - name: "Android 10 (api 29)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
       - id: Nokia_7_1_real_us # Nokia 7.1 - api 29 (10)

.sauce/sentry-uitest-android-ui.yml

@@ -19,18 +19,30 @@ espresso:
 suites:
 
   - name: "Android 13 Ui test (api 33)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_5_13_real_us # Google Pixel 5 - api 33 (13)
 
   - name: "Android 12 Ui test (api 31)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Samsung_Galaxy_S22_Ultra_5G_real_us # Samsung Galaxy S22 Ultra 5G - api 31 (12)
 
   - name: "Android 11 Ui test (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11)
 
   - name: "Android 10 Ui test (api 29)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_7T_real_us # OnePlus 7T - api 29 (10)
 

buildSrc/src/main/java/Config.kt

@@ -167,7 +167,7 @@ object Config {
         val androidxTestRules = "androidx.test:rules:$androidxTestVersion"
         val espressoCore = "androidx.test.espresso:espresso-core:$espressoVersion"
         val espressoIdlingResource = "androidx.test.espresso:espresso-idling-resource:$espressoVersion"
-        val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.1"
+        val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.2"
         val androidxJunit = "androidx.test.ext:junit:1.1.3"
         val androidxCoreKtx = "androidx.core:core-ktx:1.7.0"
         val robolectric = "org.robolectric:robolectric:4.7.3"

sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts

@@ -26,8 +26,11 @@ android {
         // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle
         // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
         // Check the readme for more info.
-        // Test orchestrator was removed due to issues with SauceLabs
-//        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+    }
+
+    testOptions {
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
     }
 
     buildFeatures {

sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts

@@ -25,8 +25,11 @@ android {
         // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle
         // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
         // Check the readme for more info.
-        // Test orchestrator was removed due to issues with SauceLabs
-//        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+    }
+
+    testOptions {
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
     }
 
     buildFeatures {

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt

@@ -12,6 +12,7 @@ import io.sentry.ProfilingTraceData
 import io.sentry.Sentry
 import io.sentry.SentryEvent
 import io.sentry.android.core.SentryAndroidOptions
+import io.sentry.assertEnvelopeItem
 import io.sentry.profilemeasurements.ProfileMeasurement
 import io.sentry.protocol.SentryTransaction
 import org.junit.runner.RunWith
@@ -34,7 +35,7 @@ class EnvelopeTests : BaseUiTest() {
         Sentry.captureMessage("Message captured during test")
 
         relay.assert {
-            assertEnvelope {
+            assertFirstEnvelope {
                 val event: SentryEvent = it.assertItem()
                 it.assertNoOtherItems()
                 assertTrue(event.message?.formatted == "Message captured during test")
@@ -53,7 +54,7 @@ class EnvelopeTests : BaseUiTest() {
 
         relayIdlingResource.increment()
         IdlingRegistry.getInstance().register(ProfilingSampleActivity.scrollingIdlingResource)
-        val transaction = Sentry.startTransaction("e2etests", "test1")
+        val transaction = Sentry.startTransaction("profiledTransaction", "test1")
         val sampleScenario = launchActivity<ProfilingSampleActivity>()
         swipeList(1)
         sampleScenario.moveToState(Lifecycle.State.DESTROYED)
@@ -63,21 +64,29 @@ class EnvelopeTests : BaseUiTest() {
 
         transaction.finish()
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "ProfilingSampleActivity"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals("ProfilingSampleActivity", transactionItem.transaction)
             }
-            assertEnvelope {
+
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "profiledTransaction"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", transactionItem.transaction)
+                assertEquals("profiledTransaction", transactionItem.transaction)
             }
-            assertEnvelope {
+
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "profiledTransaction"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(profilingTraceData.transactionId, transaction.eventId.toString())
-                assertEquals("e2etests", profilingTraceData.transactionName)
+                assertEquals("profiledTransaction", profilingTraceData.transactionName)
                 assertTrue(profilingTraceData.environment.isNotEmpty())
                 assertTrue(profilingTraceData.cpuArchitecture.isNotEmpty())
                 assertTrue(profilingTraceData.transactions.isNotEmpty())
@@ -127,24 +136,31 @@ class EnvelopeTests : BaseUiTest() {
         transaction3.finish()
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction.eventId.toString(), transactionItem.eventId.toString())
             }
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests1"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction2.eventId.toString(), transactionItem.eventId.toString())
             }
-            // The profile is sent only in the last transaction envelope
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests2"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction3.eventId.toString(), transactionItem.eventId.toString())
             }
-            // The profile is sent only in the last transaction envelope
-            assertEnvelope {
+            // The profile is sent in its own envelope
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "e2etests2"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals("e2etests2", profilingTraceData.transactionName)
@@ -191,7 +207,7 @@ class EnvelopeTests : BaseUiTest() {
         relayIdlingResource.increment()
         relayIdlingResource.increment()
         val profilesDirPath = Sentry.getCurrentHub().options.profilingTracesDirPath
-        val transaction = Sentry.startTransaction("e2etests", "test empty")
+        val transaction = Sentry.startTransaction("emptyProfileTransaction", "test empty")
 
         var finished = false
         Thread {
@@ -209,13 +225,15 @@ class EnvelopeTests : BaseUiTest() {
         }
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "emptyProfileTransaction"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", transactionItem.transaction)
+                assertEquals("emptyProfileTransaction", transactionItem.transaction)
             }
             // The profile failed to be sent. Trying to read the envelope from the data transmitted throws an exception
-            assertFailsWith<IllegalArgumentException> { assertEnvelope {} }
+            assertFailsWith<IllegalArgumentException> { assertFirstEnvelope {} }
             assertNoOtherEnvelopes()
             assertNoOtherRequests()
         }
@@ -230,14 +248,16 @@ class EnvelopeTests : BaseUiTest() {
             options.profilesSampleRate = 1.0
         }
         relayIdlingResource.increment()
-        Sentry.startTransaction("e2etests", "testTimeout")
+        Sentry.startTransaction("timedOutProfile", "testTimeout")
         // We don't call transaction.finish() and let the timeout do its job
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "timedOutProfile"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", profilingTraceData.transactionName)
+                assertEquals("timedOutProfile", profilingTraceData.transactionName)
                 assertEquals(ProfilingTraceData.TRUNCATION_REASON_TIMEOUT, profilingTraceData.truncationReason)
             }
             assertNoOtherEnvelopes()

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt

@@ -2,8 +2,10 @@ package io.sentry.uitest.android.mockservers
 
 import io.sentry.EnvelopeReader
 import io.sentry.Sentry
+import io.sentry.SentryEnvelope
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.RecordedRequest
+import java.io.IOException
 import java.util.zip.GZIPInputStream
 
 /** Class used to assert requests sent to [MockRelay]. */
@@ -13,19 +15,17 @@ class RelayAsserter(
 ) {
 
     /**
-     * Asserts an envelope request exists and allows to make other assertions on it and its response.
-     * The asserted envelope request is then removed from internal list of unasserted envelope.
+     * Asserts an envelope request exists and allows to make other assertions on the first one and on its response.
+     * The asserted envelope request is then removed from internal list of unasserted envelopes.
      */
-    fun assertRawEnvelope(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) {
+    fun assertFirstRawEnvelope(assertion: ((relayResponse: RelayResponse) -> Unit)? = null) {
         val relayResponse = unassertedEnvelopes.removeFirstOrNull()
             ?: throw AssertionError("No envelope request found")
-        assertion?.let {
-            it(relayResponse.request, relayResponse.response)
-        }
+        assertion?.let { it(relayResponse) }
     }
 
     /**
-     * Asserts a request exists and makes other assertions on it and its response.
+     * Asserts a request exists and makes other assertions on the first one and on its response.
      * The asserted request is then removed from internal list of unasserted requests.
      */
     fun assertRawRequest(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) {
@@ -37,19 +37,27 @@ class RelayAsserter(
     }
 
     /**
-     * Asserts a request exists, parses it as an envelope and makes other assertions through a [EnvelopeAsserter].
+     * Parses the first request as an envelope and makes other assertions through a [EnvelopeAsserter].
      * The asserted envelope is then removed from internal list of unasserted envelopes.
      */
-    fun assertEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) {
-        assertRawEnvelope { request, response ->
-            // Parse the request to rebuild the original envelope. If it fails we throw an assertion error.
-            val envelope = EnvelopeReader(Sentry.getCurrentHub().options.serializer)
-                .read(GZIPInputStream(request.body.inputStream()))
-                ?: throw AssertionError("Was unable to parse the request as an envelope: $request")
-            assertion(EnvelopeAsserter(envelope, response))
+    fun assertFirstEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) {
+        assertFirstRawEnvelope { relayResponse ->
+            relayResponse.assert(assertion)
         }
     }
 
+    /**
+     * Returns the first request that can be parsed as an envelope and that satisfies [filter].
+     * Throws an [AssertionError] if the envelope was not found.
+     */
+    fun findEnvelope(
+        filter: (envelope: SentryEnvelope) -> Boolean = { true }
+    ): RelayResponse {
+        val relayResponseIndex = unassertedEnvelopes.indexOfFirst { it.envelope?.let(filter) ?: false }
+        if (relayResponseIndex == -1) throw AssertionError("No envelope request found with specified filter")
+        return unassertedEnvelopes.removeAt(relayResponseIndex)
+    }
+
     /** Asserts no other envelopes were sent. */
     fun assertNoOtherEnvelopes() {
         if (unassertedEnvelopes.isNotEmpty()) {
@@ -71,5 +79,24 @@ class RelayAsserter(
         assertNoOtherRawRequests()
     }
 
-    data class RelayResponse(val request: RecordedRequest, val response: MockResponse)
+    data class RelayResponse(val request: RecordedRequest, val response: MockResponse) {
+
+        /** Request parsed as envelope. */
+        val envelope: SentryEnvelope? by lazy {
+            try {
+                EnvelopeReader(Sentry.getCurrentHub().options.serializer)
+                    .read(GZIPInputStream(request.body.inputStream()))
+            } catch (e: IOException) {
+                null
+            }
+        }
+
+        /** Run [assertion] on this request parsed as an envelope. */
+        fun assert(assertion: (asserter: EnvelopeAsserter) -> Unit) {
+            // Parse the request to rebuild the original envelope. If it fails we throw an assertion error.
+            envelope?.let {
+                assertion(EnvelopeAsserter(it, response))
+            } ?: throw AssertionError("Was unable to parse the request as an envelope: $request")
+        }
+    }
 }

sentry-test-support/src/main/kotlin/io/sentry/Assertions.kt

@@ -32,7 +32,10 @@ fun checkTransaction(predicate: (SentryTransaction) -> Unit): SentryEnvelope {
 /**
  * Asserts an envelope item of [T] exists in [items] and returns the first one. Otherwise it throws an [AssertionError].
  */
-inline fun <reified T> assertEnvelopeItem(items: List<SentryEnvelopeItem>, predicate: (index: Int, item: T) -> Unit): T {
+inline fun <reified T> assertEnvelopeItem(
+    items: List<SentryEnvelopeItem>,
+    predicate: (index: Int, item: T) -> Unit = { _, _ -> }
+): T {
     val item = items.mapIndexedNotNull { index, it ->
         val deserialized = JsonSerializer(SentryOptions()).deserialize(it.data.inputStream().reader(), T::class.java)
         deserialized?.let { Pair(index, it) }