Fix UI tests (#4692)

* moved lazy field load after old executor service shutdown
* wrapped SystemEventsBreadcrumbsIntegration.unregisterReceiver in try catch

* Run command to disable system animations
* upgraded androidx test and espresso dependencies
* removed saucelabs test devices (now run UI tests on 1 device only)
* Added SentryEnvelope.describeForTest() to BaseUiTest
* Added sent envelopes to assert exception description

* removed frame timestamp check, as useless in that case
* fixed doubleInitWithSameOption UI test by waiting for the last envelope
* fixed UserFeedbackUiTest.userFeedbackSendEnvelope UI test by checking the replay was sent, too

Author: stefanosiano

.sauce/sentry-uitest-android-ui.yml

@@ -26,22 +26,6 @@ suites:
       - name: ".*"
         platformVersion: "15"
 
-  - name: "Android 14 Ui test (api 34)"
-    testOptions:
-      clearPackageData: true
-      useTestOrchestrator: true
-    devices:
-      - name: ".*"
-        platformVersion: "14"
-
-  - name: "Android 13 Ui test (api 33)"
-    testOptions:
-      clearPackageData: true
-      useTestOrchestrator: true
-    devices:
-      - name: ".*"
-        platformVersion: "13"
-
 # Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
 artifacts:
   download:

gradle/libs.versions.toml

@@ -2,11 +2,11 @@
 apollo = "2.5.9"
 androidxLifecycle = "2.2.0"
 androidxNavigation = "2.4.2"
-androidxTestCore = "1.6.1"
+androidxTestCore = "1.7.0"
 androidxCompose = "1.6.3"
 composeCompiler = "1.5.14"
 coroutines = "1.6.1"
-espresso = "3.5.0"
+espresso = "3.7.0"
 feign = "11.6"
 jacoco = "0.8.7"
 jackson = "2.18.3"

sentry-android-core/src/main/java/io/sentry/android/core/SystemEventsBreadcrumbsIntegration.java

@@ -46,6 +46,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -181,26 +182,30 @@ private void registerReceiver(
     }
   }
 
-  private void unregisterReceiver() {
+  @SuppressWarnings("Convert2MethodRef") // older AGP versions do not support method references
+  private void scheduleUnregisterReceiver() {
     if (options == null) {
       return;
     }
 
-    options
-        .getExecutorService()
-        .submit(
-            () -> {
-              final @Nullable SystemEventsBroadcastReceiver receiverRef;
-              try (final @NotNull ISentryLifecycleToken ignored = receiverLock.acquire()) {
-                isStopped = true;
-                receiverRef = receiver;
-                receiver = null;
-              }
+    try {
+      options.getExecutorService().submit(() -> unregisterReceiver());
+    } catch (RejectedExecutionException e) {
+      unregisterReceiver();
+    }
+  }
 
-              if (receiverRef != null) {
-                context.unregisterReceiver(receiverRef);
-              }
-            });
+  private void unregisterReceiver() {
+    final @Nullable SystemEventsBroadcastReceiver receiverRef;
+    try (final @NotNull ISentryLifecycleToken ignored = receiverLock.acquire()) {
+      isStopped = true;
+      receiverRef = receiver;
+      receiver = null;
+    }
+
+    if (receiverRef != null) {
+      context.unregisterReceiver(receiverRef);
+    }
   }
 
   @Override
@@ -215,7 +220,7 @@ public void close() throws IOException {
     }
 
     AppState.getInstance().removeAppStateListener(this);
-    unregisterReceiver();
+    scheduleUnregisterReceiver();
 
     if (options != null) {
       options.getLogger().log(SentryLevel.DEBUG, "SystemEventsBreadcrumbsIntegration removed.");
@@ -264,7 +269,7 @@ public void onForeground() {
 
   @Override
   public void onBackground() {
-    unregisterReceiver();
+    scheduleUnregisterReceiver();
   }
 
   final class SystemEventsBroadcastReceiver extends BroadcastReceiver {

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt

@@ -8,10 +8,17 @@ import androidx.test.espresso.IdlingRegistry
 import androidx.test.espresso.idling.CountingIdlingResource
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.runner.AndroidJUnitRunner
+import io.sentry.JsonSerializer
+import io.sentry.ProfilingTraceData
 import io.sentry.Sentry
 import io.sentry.Sentry.OptionsConfiguration
+import io.sentry.SentryEnvelope
+import io.sentry.SentryEvent
+import io.sentry.SentryItemType
+import io.sentry.SentryOptions
 import io.sentry.android.core.SentryAndroid
 import io.sentry.android.core.SentryAndroidOptions
+import io.sentry.protocol.SentryTransaction
 import io.sentry.test.applyTestOptions
 import io.sentry.test.initForTest
 import io.sentry.uitest.android.mockservers.MockRelay
@@ -43,9 +50,9 @@ abstract class BaseUiTest {
   /** Mock relay server that receives all envelopes sent during the test. */
   protected val relay = MockRelay(false, relayIdlingResource)
 
-  private fun disableDontKeepActivities() {
+  private fun runCommand(cmd: String) {
     val automation = InstrumentationRegistry.getInstrumentation().uiAutomation
-    val pfd = automation.executeShellCommand("settings put global always_finish_activities 0")
+    val pfd = automation.executeShellCommand(cmd)
     try {
       FileInputStream(pfd.fileDescriptor).readBytes()
     } catch (e: Throwable) {
@@ -54,6 +61,16 @@ abstract class BaseUiTest {
     pfd.close()
   }
 
+  private fun disableDontKeepActivities() {
+    runCommand("settings put global always_finish_activities 0")
+  }
+
+  fun disableSystemAnimations() {
+    runCommand("settings put global window_animation_scale 0")
+    runCommand("settings put global transition_animation_scale 0")
+    runCommand("settings put global animator_duration_scale 0")
+  }
+
   @BeforeTest
   fun baseSetUp() {
     runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner
@@ -63,6 +80,7 @@ abstract class BaseUiTest {
     relay.start()
     mockDsn = relay.createMockDsn()
     disableDontKeepActivities()
+    disableSystemAnimations()
   }
 
   @AfterTest
@@ -126,3 +144,40 @@ fun initForTest(
     optionsConfiguration.configure(it)
   }
 }
+
+/**
+ * Function used to describe the content of the envelope to print in the logs. For debugging
+ * purposes only.
+ */
+internal fun SentryEnvelope.describeForTest(): String {
+  var descr = ""
+  items.forEach { item ->
+    when (item.header.type) {
+      SentryItemType.Event -> {
+        val deserialized =
+          JsonSerializer(SentryOptions())
+            .deserialize(item.data.inputStream().reader(), SentryEvent::class.java)!!
+        descr +=
+          "Event (${deserialized.eventId}) - message: ${deserialized.message!!.formatted} -- "
+      }
+      SentryItemType.Transaction -> {
+        val deserialized =
+          JsonSerializer(SentryOptions())
+            .deserialize(item.data.inputStream().reader(), SentryTransaction::class.java)!!
+        descr +=
+          "Transaction (${deserialized.eventId}) - transaction: ${deserialized.transaction} - spans: ${deserialized.spans.joinToString { "${it.op} ${it.description}" }} -- "
+      }
+      SentryItemType.Profile -> {
+        val deserialized =
+          JsonSerializer(SentryOptions())
+            .deserialize(item.data.inputStream().reader(), ProfilingTraceData::class.java)!!
+        descr +=
+          "Profile (${deserialized.profileId}) - transactionName: ${deserialized.transactionName} -- "
+      }
+      else -> {
+        descr += "${item.header.type} -- "
+      }
+    }
+  }
+  return "*** Envelope: $descr ***"
+}

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt

@@ -56,7 +56,6 @@ class EnvelopeTests : BaseUiTest() {
 
     relayIdlingResource.increment()
     IdlingRegistry.getInstance().register(ProfilingSampleActivity.scrollingIdlingResource)
-    Thread.sleep(1000)
     val transaction = Sentry.startTransaction("profiledTransaction", "test1")
     val sampleScenario = launchActivity<ProfilingSampleActivity>()
     swipeList(1)
@@ -171,15 +170,17 @@ class EnvelopeTests : BaseUiTest() {
                   values.last().relativeStartNs.toLong() <= maxTimestampAllowed,
                   "Last measurement value for '$name' is outside bounds (was: ${values.last().relativeStartNs.toLong()}ns, max: ${maxTimestampAllowed}ns",
                 )
-              }
 
-              // Timestamps of measurements should differ at least 10 milliseconds from each other
-              (1 until values.size).forEach { i ->
-                assertTrue(
-                  values[i].relativeStartNs.toLong() >=
-                    values[i - 1].relativeStartNs.toLong() + TimeUnit.MILLISECONDS.toNanos(10),
-                  "Measurement value timestamp for '$name' does not differ at least 10ms",
-                )
+                // Timestamps of measurements should differ at least 10 milliseconds from each other
+                (1 until values.size).forEach { i ->
+                  val measurementTimestampDiff =
+                    values[i].relativeStartNs.toLong() - values[i - 1].relativeStartNs.toLong()
+
+                  assertTrue(
+                    measurementTimestampDiff >= TimeUnit.MILLISECONDS.toNanos(10),
+                    "Measurement value timestamp for '$name' should differ at least 10ms, but was $measurementTimestampDiff",
+                  )
+                }
               }
             }
 

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/SdkInitTests.kt

@@ -56,13 +56,16 @@ class SdkInitTests : BaseUiTest() {
     }
     val transaction = Sentry.startTransaction("e2etests", "testInit")
     val sampleScenario = launchActivity<EmptyActivity>()
+
     initSentry(true) {
       it.tracesSampleRate = 1.0
       it.profilesSampleRate = 1.0
       // We use the same executorService before and after closing the SDK
       it.executorService = options.executorService
       it.isDebug = true
     }
+
+    relayIdlingResource.increment()
     relayIdlingResource.increment()
     relayIdlingResource.increment()
     transaction.finish()
@@ -79,9 +82,19 @@ class SdkInitTests : BaseUiTest() {
           it.assertNoOtherItems()
           assertEquals("e2etests", transactionItem.transaction)
         }
-    }
 
-    relay.assert {
+      findEnvelope {
+          assertEnvelopeTransaction(it.items.toList(), AndroidLogger()).transaction ==
+            "EmptyActivity"
+        }
+        .assert {
+          val transactionItem: SentryTransaction = it.assertTransaction()
+          // Transaction-based Profiling is already in e2etests transaction, so it won't run again
+          // here
+          it.assertNoOtherItems()
+          assertEquals("EmptyActivity", transactionItem.transaction)
+        }
+
       findEnvelope {
           assertEnvelopeTransaction(it.items.toList(), AndroidLogger()).transaction == "e2etests2"
         }

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt

@@ -23,6 +23,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
 import io.sentry.Sentry
 import io.sentry.SentryEvent
 import io.sentry.SentryFeedbackOptions.SentryFeedbackCallback
+import io.sentry.SentryItemType
 import io.sentry.SentryOptions
 import io.sentry.android.core.AndroidLogger
 import io.sentry.android.core.R
@@ -471,6 +472,9 @@ class UserFeedbackUiTest : BaseUiTest() {
       //  because it would block the espresso interactions (button click)
       it.feedbackOptions.onSubmitSuccess = SentryFeedbackCallback {
         relayIdlingResource.increment()
+        if (enableReplay) {
+          relayIdlingResource.increment()
+        }
       }
       // Let's capture a replay, so we can check the replayId in the feedback
       if (enableReplay) {
@@ -511,6 +515,10 @@ class UserFeedbackUiTest : BaseUiTest() {
             )
           }
         }
+      if (enableReplay) {
+        findEnvelope { it.items.first().header.type == SentryItemType.ReplayVideo }
+      }
+      assertNoOtherEnvelopes()
     }
   }
 

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt

@@ -1,6 +1,7 @@
 package io.sentry.uitest.android.mockservers
 
 import androidx.test.espresso.idling.CountingIdlingResource
+import io.sentry.uitest.android.describeForTest
 import io.sentry.uitest.android.waitUntilIdle
 import kotlin.test.assertNotNull
 import okhttp3.mockwebserver.Dispatcher
@@ -118,7 +119,19 @@ class MockRelay(
   /** Wait to receive all requests (if [waitForRequests] is true) and run the [assertion]. */
   fun assert(assertion: RelayAsserter.() -> Unit) {
     if (waitForRequests) {
-      waitUntilIdle()
+      try {
+        waitUntilIdle()
+      } catch (e: Exception) {
+        if (unassertedEnvelopes.isNotEmpty()) {
+          throw AssertionError(
+            "There was a total of ${unassertedEnvelopes.size} envelopes: " +
+              unassertedEnvelopes.joinToString { it.envelope!!.describeForTest() },
+            e,
+          )
+        } else {
+          throw e
+        }
+      }
     }
     assertion(RelayAsserter(unassertedEnvelopes))
   }