Adding benchmark to run as instrumentation tests for builds (#1999)

* adding benchmark to instrumentation tests * setting benchmark test build type to debug * removing benchmark runner * suppress benchmarking errors * spotless checks * spotless corrections applied * build type to debug * disable dry run mode * spotless corrections applied * testing benchmark with release build * release and debug package: * bumping up benchmakr junit version * enabling dry test mode * spotless corrections applied * spotless corrections applied * trying with different sdk version * benchmark running on FTL * spotless corrections * adding config for storage * correcting kokoro build * configuring external storage for benchmark results * removing extra arguments * running spotless corrections * increaseing test timeout * reducing the number of patients for sync benchmark * test arguments modification
google · May 22, 2023 · b2b6dc5 · b2b6dc5
1 parent e611a5d
commit b2b6dc5
Show file tree

Hide file tree

Showing 9 changed files with 101 additions and 46 deletions.
diff --git a/benchmark/build.gradle.kts b/benchmark/build.gradle.kts
@@ -24,10 +24,7 @@ android {
  targetSdk = Sdk.targetSdk
 
  testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner"
- // Runs only once
- testInstrumentationRunnerArguments["androidx.benchmark.dryRunMode.enable"] = "true"
- // Includes Startup time
- testInstrumentationRunnerArguments["androidx.benchmark.startupMode.enable"] = "true"
+ testInstrumentationRunnerArguments["androidx.benchmark.output.enable"] = "true"
  }
 
  testBuildType = "release"
@@ -68,6 +65,8 @@ android {
  }
 }
 
+afterEvaluate { configureFirebaseTestLabForMicroBenchmark() }
+
 configurations {
  all {
  removeIncompatibleDependencies()

diff --git a/benchmark/src/androidTest/AndroidManifest.xml b/benchmark/src/androidTest/AndroidManifest.xml
@@ -16,6 +16,7 @@
  android:usesCleartextTraffic="true"
  tools:ignore="HardcodedDebugMode"
  tools:replace="android:debuggable"
+ android:requestLegacyExternalStorage="true"
  >
  <profileable android:shell="true" />
  </application>

diff --git a/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/D_FhirJsonParserBenchmark.kt b/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/D_FhirJsonParserBenchmark.kt
@@ -43,7 +43,7 @@ class D_FhirJsonParserBenchmark {
  fun parseLightFhirBundle() {
  benchmarkRule.measureRepeated {
  val jsonParser = runWithTimingDisabled {
- var fhirContext = FhirContext.forCached(FhirVersionEnum.R4)
+ val fhirContext = FhirContext.forCached(FhirVersionEnum.R4)
  fhirContext.newJsonParser()
  }
 
@@ -60,7 +60,7 @@ class D_FhirJsonParserBenchmark {
  fun parseLightFhirLibrary() {
  benchmarkRule.measureRepeated {
  val jsonParser = runWithTimingDisabled {
- var fhirContext = FhirContext.forCached(FhirVersionEnum.R4)
+ val fhirContext = FhirContext.forCached(FhirVersionEnum.R4)
  fhirContext.newJsonParser()
  }
 

diff --git a/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/F_EngineDatabaseBenchmark.kt b/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/F_EngineDatabaseBenchmark.kt
@@ -22,12 +22,15 @@ import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import ca.uhn.fhir.context.FhirContext
 import ca.uhn.fhir.context.FhirVersionEnum
+import com.google.android.fhir.FhirEngineConfiguration
 import com.google.android.fhir.FhirEngineProvider
 import com.google.common.truth.Truth.assertThat
 import java.io.InputStream
 import kotlinx.coroutines.runBlocking
 import org.hl7.fhir.r4.model.Bundle
 import org.hl7.fhir.r4.model.ResourceType
+import org.junit.AfterClass
+import org.junit.BeforeClass
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -63,4 +66,18 @@ class F_EngineDatabaseBenchmark {
  }
  }
  }
+
+ companion object {
+
+ @JvmStatic
+ @BeforeClass
+ fun oneTimeSetup() {
+ FhirEngineProvider.init(FhirEngineConfiguration(testMode = true))
+ }
+ @JvmStatic
+ @AfterClass
+ fun oneTimeTearDown() {
+ FhirEngineProvider.cleanup()
+ }
+ }
 }
diff --git a/benchmark/src/androidTest/java/com/google/android/fhir/benchmark/G_CqlEvaluatorBenchmark.kt b/benchmark/src/androidTest/java/com/google/android/fhir/benchmark/G_CqlEvaluatorBenchmark.kt
@@ -23,6 +23,7 @@ import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import ca.uhn.fhir.context.FhirContext
 import ca.uhn.fhir.context.FhirVersionEnum
+import com.google.android.fhir.FhirEngineConfiguration
 import com.google.android.fhir.FhirEngineProvider
 import com.google.android.fhir.knowledge.KnowledgeManager
 import com.google.android.fhir.workflow.FhirOperatorBuilder
@@ -33,6 +34,8 @@ import kotlinx.coroutines.runBlocking
 import org.hl7.fhir.r4.model.Bundle
 import org.hl7.fhir.r4.model.Library
 import org.hl7.fhir.r4.model.Parameters
+import org.junit.AfterClass
+import org.junit.BeforeClass
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -88,4 +91,18 @@ class G_CqlEvaluatorBenchmark {
  assertThat(results.getParameterBool("CompletedImmunization")).isTrue()
  }
  }
+
+ companion object {
+
+ @JvmStatic
+ @BeforeClass
+ fun oneTimeSetup() {
+ FhirEngineProvider.init(FhirEngineConfiguration(testMode = true))
+ }
+ @JvmStatic
+ @AfterClass
+ fun oneTimeTearDown() {
+ FhirEngineProvider.cleanup()
+ }
+ }
 }
diff --git a/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/H_FhirSyncWorkerBenchmark.kt b/...hmark/src/androidTest/java/com/google/android/fhir/benchmark/H_FhirSyncWorkerBenchmark.kt
@@ -62,8 +62,7 @@ import org.hl7.fhir.r4.model.Reference
 import org.hl7.fhir.r4.model.Resource
 import org.hl7.fhir.r4.model.ResourceType
 import org.hl7.fhir.r4.model.StringType
-import org.junit.After
-import org.junit.Before
+import org.junit.AfterClass
 import org.junit.BeforeClass
 import org.junit.Rule
 import org.junit.Test
@@ -76,8 +75,6 @@ class H_FhirSyncWorkerBenchmark {
 
  private val fhirJsonParser: IParser = FhirContext.forCached(FhirVersionEnum.R4).newJsonParser()
 
- private lateinit var mockWebServer: MockWebServer
-
  class BenchmarkTestOneTimeSyncWorker(
  private val appContext: Context,
  workerParams: WorkerParameters
@@ -123,22 +120,11 @@ class H_FhirSyncWorkerBenchmark {
  }
  }
 
- @Before
- fun setup() {
- mockWebServer = MockWebServer()
- mockWebServer.start(mockServerPort)
- }
-
- @After
- fun teardown() {
- mockWebServer.shutdown()
- }
-
- @Test fun oneTimeSync_10patients() = oneTimeSync(10, 10, 10)
+ @Test fun oneTimeSync_1patient() = oneTimeSync(1, 5, 5)
 
- @Test fun oneTimeSync_100patients() = oneTimeSync(100, 10, 10)
+ @Test fun oneTimeSync_10patients() = oneTimeSync(10, 5, 5)
 
- @Test fun oneTimeSync_1000patients() = oneTimeSync(1000, 10, 10)
+ @Test fun oneTimeSync_50patients() = oneTimeSync(50, 5, 5)
 
  private fun oneTimeSync(numberPatients: Int, numberObservations: Int, numberEncounters: Int) =
  runBlocking {
@@ -403,6 +389,7 @@ class H_FhirSyncWorkerBenchmark {
  companion object {
 
  private const val mockServerPort = 8080
+ private val mockWebServer: MockWebServer = MockWebServer()
 
  @JvmStatic
  @BeforeClass
@@ -413,6 +400,14 @@ class H_FhirSyncWorkerBenchmark {
  testMode = true
  )
  )
+ mockWebServer.start(mockServerPort)
+ }
+
+ @JvmStatic
+ @AfterClass
+ fun oneTimeTearDown() {
+ FhirEngineProvider.cleanup()
+ mockWebServer.shutdown()
  }
  }
 }
diff --git a/buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt b/buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt
@@ -15,6 +15,7 @@
  */
 
 import com.android.build.api.dsl.LibraryExtension
+import com.osacky.flank.gradle.FlankGradleExtension
 import java.util.UUID
 import org.gradle.api.Project
 import org.gradle.kotlin.dsl.apply
@@ -23,45 +24,68 @@ import org.gradle.kotlin.dsl.configure
 @Suppress("SdCardPath")
 fun Project.configureFirebaseTestLab() {
  apply(plugin = Plugins.BuildPlugins.fladle)
- configure<com.osacky.flank.gradle.FlankGradleExtension> {
- projectId.set("android-fhir-instrumeted-tests")
- debugApk.set(
- project.provider {
- "$rootDir/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
- }
+ configure<FlankGradleExtension> {
+ commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLab)
+ instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/debug/*.apk" })
+ environmentVariables.set(
+ mapOf("coverage" to "true", "coverageFile" to "/sdcard/Download/coverage.ec")
  )
+ flakyTestAttempts.set(3)
  devices.set(
  listOf(
  mapOf(
  "model" to "Nexus6P",
  "version" to
- "${this@configureFirebaseTestLab.extensions.getByType(LibraryExtension::class.java).defaultConfig.minSdk}",
+ "${project.extensions.getByType(LibraryExtension::class.java).defaultConfig.minSdk}",
  "locale" to "en_US"
  ),
  mapOf("model" to "Nexus6P", "version" to "27", "locale" to "en_US"),
  mapOf(
  "model" to "oriole",
  "version" to
- "${this@configureFirebaseTestLab.extensions.getByType(LibraryExtension::class.java).defaultConfig.targetSdk}",
+ "${project.extensions.getByType(LibraryExtension::class.java).defaultConfig.targetSdk}",
  "locale" to "en_US"
  ),
  )
  )
- instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/debug/*.apk" })
- useOrchestrator.set(false)
- flakyTestAttempts.set(3)
+ }
+}
+
+fun Project.configureFirebaseTestLabForMicroBenchmark() {
+ apply(plugin = Plugins.BuildPlugins.fladle)
+ configure<FlankGradleExtension> {
+ commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLabForMicroBenchmark)
+ instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/release/*.apk" })
  environmentVariables.set(
- mapOf("coverage" to "true", "coverageFile" to "/sdcard/Download/coverage.ec")
+ mapOf("additionalTestOutputDir" to "/sdcard/Download", "no-isolated-storage" to "true")
  )
- directoriesToPull.set(listOf("/sdcard/Download"))
- filesToDownload.set(listOf(".*/sdcard/Download/.*.ec"))
- resultsBucket.set("android-fhir-build-artifacts")
- resultsDir.set(
- if (project.providers.environmentVariable("KOKORO_BUILD_ARTIFACTS_SUBDIR").isPresent) {
- "${System.getenv("KOKORO_BUILD_ARTIFACTS_SUBDIR")}/firebase/${project.name}"
- } else {
- "${project.name}-${UUID.randomUUID()}"
- }
+ maxTestShards.set(4)
+ // some of the benchmark tests get timed-out in the default 15m
+ testTimeout.set("30m")
+ devices.set(
+ listOf(
+ mapOf("model" to "oriole", "version" to "32", "locale" to "en_US"),
+ )
  )
  }
 }
+
+private fun FlankGradleExtension.commonConfigurationForFirebaseTestLab(project: Project) {
+ projectId.set("android-fhir-instrumeted-tests")
+ debugApk.set(
+ project.provider {
+ "${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
+ }
+ )
+ useOrchestrator.set(false)
+ directoriesToPull.set(listOf("/sdcard/Download"))
+ filesToDownload.set(listOf(".*/sdcard/Download/.*.ec", ".*/sdcard/Download/.*.json"))
+ resultsBucket.set("android-fhir-build-artifacts")
+ resultsDir.set(
+ if (project.providers.environmentVariable("KOKORO_BUILD_ARTIFACTS_SUBDIR").isPresent) {
+ "${System.getenv("KOKORO_BUILD_ARTIFACTS_SUBDIR")}/firebase/${project.name}"
+ } else {
+ "${project.name}-${UUID.randomUUID()}"
+ }
+ )
+}
diff --git a/knowledge/build.gradle.kts b/knowledge/build.gradle.kts
@@ -95,6 +95,7 @@ dependencies {
  implementation(Dependencies.Room.ktx)
  implementation(Dependencies.Room.runtime)
  implementation(Dependencies.timber)
+ implementation(Dependencies.Kotlin.kotlinCoroutinesCore)
 
  kapt(Dependencies.Room.compiler)
 

diff --git a/kokoro/gcp_ubuntu/kokoro_build.sh b/kokoro/gcp_ubuntu/kokoro_build.sh
@@ -90,7 +90,8 @@ function build_only() {
 # coverage reports.
 function device_tests() {
  ./gradlew packageDebugAndroidTest --scan --stacktrace
- local lib_names=("datacapture" "engine" "workflow")
+ ./gradlew packageReleaseAndroidTest --scan --stacktrace
+ local lib_names=("datacapture" "engine" "workflow" "benchmark")
  firebase_pids=()
  for lib_name in "${lib_names[@]}"; do
  ./gradlew :$lib_name:runFlank --scan --stacktrace &