test: fix functional test flakes

embrace-io · Mar 7, 2024 · 81c5f04 · 81c5f04
1 parent dbc32b8
commit 81c5f04
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 76 deletions.
diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/AnrIntegrationTest.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/AnrIntegrationTest.kt
@@ -75,18 +75,18 @@ internal class AnrIntegrationTest : BaseTest() {
  sendBackground()
 
  // ignore startup moment end request that is validated in other tests
- waitForRequest()
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) {})
 
  // validate ANRs with JUnit assertions rather than golden file
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.SESSIONS) { request ->
  val payload = readBodyAsSessionMessage(request)
  assertNotNull(payload)
  val perfInfo by lazy { serializer.toJson(payload.performanceInfo) }
  val intervals = checkNotNull(payload.performanceInfo?.anrIntervals) {
  "No ANR intervals in payload. p=$perfInfo"
  }
  validateIntervals(intervals)
- }
+ })
  }
 
  private fun validateIntervals(intervals: List<AnrInterval>) {

diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/BaseTest.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/BaseTest.kt
@@ -99,7 +99,8 @@ internal open class BaseTest {
  }
 
  private fun getDefaultNetworkResponses(): Map<String, TestServerResponse> {
- val configMockResponse = TestServerResponse(HttpURLConnection.HTTP_OK, serializer.toJson(remoteConfig))
+ val configMockResponse =
+ TestServerResponse(HttpURLConnection.HTTP_OK, serializer.toJson(remoteConfig))
 
  return mapOf(
  EmbraceEndpoint.LOGGING.url to TestServerResponse(HttpURLConnection.HTTP_OK),
@@ -196,34 +197,27 @@ internal open class BaseTest {
  logTestMessage("Starting validation of initialization requests")
  var isStartupStartEventValidated = false
 
- (0 until TOTAL_REQUESTS_AT_INIT).forEach { _ ->
- waitForRequest { request ->
- when (request.path?.substringBefore("?")) {
- EmbraceEndpoint.EVENTS.url -> {
- assertEquals("POST", request.method)
- if (!isStartupStartEventValidated) {
- isStartupStartEventValidated = true
- validateMessageAgainstGoldenFile(
- request,
- "moment-startup-start-event.json"
- )
- } else {
- validateMessageAgainstGoldenFile(
- request,
- "moment-startup-late-event.json"
- )
- }
+ waitForRequest(
+ listOf(
+ RequestValidator(EmbraceEndpoint.EVENTS) { request ->
+ assertEquals("POST", request.method)
+ if (!isStartupStartEventValidated) {
+ isStartupStartEventValidated = true
+ validateMessageAgainstGoldenFile(
+ request,
+ "moment-startup-start-event.json"
+ )
+ } else {
+ validateMessageAgainstGoldenFile(
+ request,
+ "moment-startup-late-event.json"
+ )
  }
-
- EmbraceEndpoint.CONFIG.url -> {
- assertEquals("GET", request.method)
- }
-
- else -> handleUnexpectedFailure(request)
- }
- logTestMessage("Validated initialization request at ${request.path}")
- }
- }
+ },
+ RequestValidator(EmbraceEndpoint.CONFIG) { request ->
+ assertEquals("GET", request.method)
+ })
+ )
  }
 
  private fun handleUnexpectedFailure(request: RecordedRequest) {
@@ -241,13 +235,37 @@ internal open class BaseTest {
  * If the request is not received within a reasonable amount of time this method will
  * fail the test.
  */
- fun waitForRequest(action: (response: RecordedRequest) -> Unit = {}) {
+ fun waitForRequest(requestValidator: RequestValidator) {
+ waitForRequest(listOf(requestValidator))
+ }
+
+ fun waitForRequest(requestValidators: List<RequestValidator>) {
  logTestMessage("Waiting to assert that request was received by MockWebServer.")
- val request = testServer.takeRequest()
- request?.let(action) ?: fail(
- "Expected request not sent after configured timeout. " +
- "The SDK probably either failed to send the data or crashed - check Logcat for clues."
- )
+ var remainingCount = requestValidators.size
+
+ while (remainingCount > 0) {
+ val request = testServer.takeRequest()
+
+ if (request == null) {
+ fail(
+ "Expected request not sent after configured timeout. " +
+ "The SDK probably either failed to send the data or crashed - check Logcat for clues."
+ )
+ }
+ val req = checkNotNull(request)
+ val path = checkNotNull(req.path)
+
+ val validator = requestValidators.singleOrNull { validator ->
+ path.endsWith(validator.endpoint.url)
+ }
+
+ if (validator != null) {
+ validator.validate(req)
+ remainingCount--
+ } else {
+ logTestMessage("Unexpected request: $path")
+ }
+ }
  }
 
  fun waitForFailedRequest(
@@ -296,10 +314,13 @@ internal open class BaseTest {
 
  if (!result.success) {
  val msg by lazy {
- val observedOutput = writeFailedOutputToDisk(requestBody, goldenFileName, ".observed")
+ val observedOutput =
+ writeFailedOutputToDisk(requestBody, goldenFileName, ".observed")
  val expected =
- mContext.assets.open("golden-files/$goldenFileName").bufferedReader().readText()
- val expectedOutput = writeFailedOutputToDisk(expected, goldenFileName, ".expected")
+ mContext.assets.open("golden-files/$goldenFileName").bufferedReader()
+ .readText()
+ val expectedOutput =
+ writeFailedOutputToDisk(expected, goldenFileName, ".expected")
 
  "Request ${request.path} differs from golden-files/$goldenFileName.\n" +
  "JSON validation failure reason: ${result.message}" +

diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/LogMessageTest.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/LogMessageTest.kt
@@ -29,9 +29,9 @@ internal class LogMessageTest : BaseTest() {
  logTestMessage("Adding info log to Embrace.")
  Embrace.getInstance().logInfo("Test log info")
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-info-event.json")
- }
+ })
  }
 
  @Test
@@ -42,9 +42,9 @@ internal class LogMessageTest : BaseTest() {
  properties["info"] = "test property"
  Embrace.getInstance().logMessage("Test log info with property", Severity.INFO, properties)
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-info-with-property-event.json")
- }
+ })
  }
 
  private fun validateFileContent(file: File) {
@@ -68,9 +68,9 @@ internal class LogMessageTest : BaseTest() {
  logTestMessage("Adding error log to Embrace.")
  Embrace.getInstance().logError("Test log error")
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-error-event.json")
- }
+ })
  }
 
  @Test
@@ -81,19 +81,19 @@ internal class LogMessageTest : BaseTest() {
 
  Embrace.getInstance().logMessage("Test log error", Severity.ERROR, properties)
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-error-with-property-event.json")
- }
+ })
  }
 
  @Test
  fun logExceptionTest() {
  logTestMessage("Adding exception log to Embrace.")
  Embrace.getInstance().logException(Exception("Another log error"))
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-error-with-exception-event.json")
- }
+ })
  }
 
  @Test
@@ -102,21 +102,18 @@ internal class LogMessageTest : BaseTest() {
  val exception = java.lang.NullPointerException("Exception message")
  Embrace.getInstance().logException(exception, Severity.ERROR, mapOf(), "log message")
 
- waitForRequest { request ->
- validateMessageAgainstGoldenFile(
- request,
- "log-error-with-exception-and-message-event.json"
- )
- }
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
+ validateMessageAgainstGoldenFile(request, "log-error-with-exception-and-message-event.json")
+ })
  }
 
  @Test
  fun logWarningTest() {
  logTestMessage("Adding warning log to Embrace.")
  Embrace.getInstance().logWarning("Test log warning")
 
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.LOGGING) { request ->
  validateMessageAgainstGoldenFile(request, "log-warning-event.json")
- }
+ })
  }
 }
diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/MomentMessageTest.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/MomentMessageTest.kt
@@ -35,18 +35,18 @@ internal class MomentMessageTest : BaseTest() {
  Embrace.getInstance().startMoment(MOMENT_NAME)
 
  // Validate start moment request
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) { request ->
  validateMessageAgainstGoldenFile(request, "moment-custom-start-event.json")
- }
+ })
 
  // Send end moment
  logTestMessage("Ending start moment to Embrace.")
  Embrace.getInstance().endMoment(MOMENT_NAME)
 
  // Validate end moment request
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) { request ->
  validateMessageAgainstGoldenFile(request, "moment-custom-end-event.json")
- }
+ })
  }
 
  /**
@@ -57,7 +57,7 @@ internal class MomentMessageTest : BaseTest() {
  // ignore startup event
  logTestMessage("Ending appStartup moment in Embrace.")
  Embrace.getInstance().endAppStartup()
- waitForRequest()
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) {})
 
  logTestMessage("Sending moment in Embrace.")
  val properties = HashMap<String, Any>()
@@ -68,25 +68,24 @@ internal class MomentMessageTest : BaseTest() {
  Embrace.getInstance().startMoment(MOMENT_NAME, MOMENT_NAME, properties)
 
  // Validate start moment request with properties
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) { request ->
  validateMessageAgainstGoldenFile(
  request,
  "moment-custom-with-properties-start-event.json"
  )
- }
+ })
 
  // Send end moment
  logTestMessage("Ending moment in Embrace.")
  Embrace.getInstance().endMoment(MOMENT_NAME)
 
  // Validate end moment request
- waitForRequest { request ->
+ waitForRequest(RequestValidator(EmbraceEndpoint.EVENTS) { request ->
  validateMessageAgainstGoldenFile(
  request,
  "moment-custom-with-properties-end-event.json"
  )
- }
-
+ })
  }
 
  private fun validateFileContent(file: File) {

diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/RequestValidator.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/RequestValidator.kt
@@ -0,0 +1,8 @@
+package io.embrace.android.embracesdk
+
+import okhttp3.mockwebserver.RecordedRequest
+
+internal class RequestValidator(
+ val endpoint: EmbraceEndpoint,
+ val validate: (response: RecordedRequest) -> Unit
+)
diff --git a/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/SessionMessageTest.kt b/embrace-android-sdk/src/androidTest/java/io/embrace/android/embracesdk/SessionMessageTest.kt
@@ -20,18 +20,21 @@ internal class SessionMessageTest : BaseTest() {
  addCoreWebVitals()
  sendBackground()
 
- waitForRequest { request ->
- validateMessageAgainstGoldenFile(request, "moment-startup-end-event.json")
- }
-
- waitForRequest { request ->
- validateMessageAgainstGoldenFile(request, "session-end.json")
- }
+ waitForRequest(
+ listOf(
+ RequestValidator(EmbraceEndpoint.EVENTS) { request ->
+ validateMessageAgainstGoldenFile(request, "moment-startup-end-event.json")
+ },
+ RequestValidator(EmbraceEndpoint.SESSIONS) { request ->
+ validateMessageAgainstGoldenFile(request, "session-end.json")
+ })
+ )
  }
 
  private fun addCoreWebVitals() {
  val webViewExpectedLog =
- mContext.assets.open("golden-files/${"expected-webview-core-vital.json"}").bufferedReader().readText()
+ mContext.assets.open("golden-files/${"expected-webview-core-vital.json"}")
+ .bufferedReader().readText()
  Embrace.getInstance().trackWebViewPerformance("myWebView", webViewExpectedLog)
  }
 }