Support sending gzipped request body when uploading (#2004)

* Support sending gzipped request body

* add content-length, make a singleton, and move before http logger

* minir nit

* add benchmark

* update benchmark

* append gzip to header if header already has value

benchmark/build.gradle.kts

@@ -87,6 +87,7 @@ dependencies {
  androidTestImplementation(Dependencies.Androidx.workRuntimeKtx)
  androidTestImplementation(Dependencies.AndroidxTest.workTestingRuntimeKtx)
  androidTestImplementation(Dependencies.mockWebServer)
+ androidTestImplementation(Dependencies.Retrofit.coreRetrofit)
 
  androidTestImplementation(project(":engine"))
  androidTestImplementation(project(":knowledge"))

benchmark/src/androidTest/java/com/google/android/fhir/benchmark/I_GzipUploadInterceptorBenchmark.kt

@@ -0,0 +1,206 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.fhir.benchmark
+
+import androidx.benchmark.junit4.BenchmarkRule
+import androidx.benchmark.junit4.measureRepeated
+import ca.uhn.fhir.context.FhirContext
+import ca.uhn.fhir.context.FhirVersionEnum
+import ca.uhn.fhir.parser.IParser
+import com.google.android.fhir.NetworkConfiguration
+import com.google.android.fhir.sync.remote.FhirConverterFactory
+import com.google.android.fhir.sync.remote.GzipUploadInterceptor
+import java.math.BigDecimal
+import java.util.UUID
+import java.util.concurrent.TimeUnit
+import kotlinx.coroutines.runBlocking
+import okhttp3.OkHttpClient
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import okhttp3.mockwebserver.RecordedRequest
+import org.hl7.fhir.r4.model.Bundle
+import org.hl7.fhir.r4.model.CodeableConcept
+import org.hl7.fhir.r4.model.Coding
+import org.hl7.fhir.r4.model.Observation
+import org.hl7.fhir.r4.model.Quantity
+import org.hl7.fhir.r4.model.Reference
+import org.hl7.fhir.r4.model.Resource
+import org.junit.After
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import retrofit2.Retrofit
+import retrofit2.http.Body
+import retrofit2.http.GET
+import retrofit2.http.POST
+import retrofit2.http.Url
+
+class I_GzipUploadInterceptorBenchmark {
+
+ @get:Rule val benchmarkRule = BenchmarkRule()
+
+ private val fhirJsonParser: IParser = FhirContext.forCached(FhirVersionEnum.R4).newJsonParser()
+
+ private lateinit var mockWebServer: MockWebServer
+
+ private lateinit var httpServiceWithGzip: TestHttpService
+
+ private lateinit var httpServiceWithoutGzip: TestHttpService
+
+ @Before
+ fun setup() {
+ mockWebServer = MockWebServer()
+ mockWebServer.start(8080)
+ val url = "http:https://${mockWebServer.hostName}:${mockWebServer.port}"
+
+ httpServiceWithGzip =
+ TestHttpService.builder(url, NetworkConfiguration(uploadWithGzip = true)).build()
+
+ httpServiceWithoutGzip =
+ TestHttpService.builder(url, NetworkConfiguration(uploadWithGzip = false)).build()
+ }
+
+ @After
+ fun teardown() {
+ mockWebServer.shutdown()
+ }
+
+ @Test fun upload_10patientsWithGzip() = uploader(10, httpServiceWithGzip)
+
+ @Test fun upload_100patientsWithGzip() = uploader(100, httpServiceWithGzip)
+
+ @Test fun upload_1000patientsWithGzip() = uploader(1000, httpServiceWithGzip)
+
+ @Test fun upload_10patientsWithoutGzip() = uploader(10, httpServiceWithoutGzip)
+
+ @Test fun upload_100patientsWithoutGzip() = uploader(100, httpServiceWithoutGzip)
+
+ @Test fun upload_1000patientsWithoutGzip() = uploader(1000, httpServiceWithoutGzip)
+
+ private fun uploader(numberObservations: Int, httpService: TestHttpService) = runBlocking {
+ mockWebServer.dispatcher =
+ object : okhttp3.mockwebserver.Dispatcher() {
+ override fun dispatch(request: RecordedRequest): MockResponse {
+ val response = MockResponse().addHeader("Content-Type", "application/json; charset=utf-8")
+ return response
+ .setBody(fhirJsonParser.encodeResourceToString(Bundle()))
+ .setResponseCode(200)
+ }
+ }
+
+ benchmarkRule.measureRepeated {
+ runBlocking {
+ val requestBundle =
+ Bundle().apply {
+ type = Bundle.BundleType.SEARCHSET
+ id = UUID.randomUUID().toString()
+ entry =
+ (1..numberObservations).map {
+ Bundle.BundleEntryComponent().setResource(createMockObservation())
+ }
+ }
+ httpService.post(requestBundle)
+ }
+ }
+ }
+
+ private fun createMockObservation(): Observation =
+ Observation().apply {
+ id = UUID.randomUUID().toString()
+ status = Observation.ObservationStatus.FINAL
+ subject = Reference("Patient/123")
+ performer = listOf(Reference("Practitioner/${UUID.randomUUID()}"))
+ value =
+ Quantity().apply {
+ value = BigDecimal.valueOf(6.2)
+ unit = "kPa"
+ code = "kPa"
+ system = "http:https://unitsofmeasure.org"
+ }
+ code =
+ CodeableConcept().apply {
+ coding = listOf(Coding("http:https://unitsofmeasure.org", "kPa", "kPa"))
+ }
+ referenceRange =
+ listOf(
+ Observation.ObservationReferenceRangeComponent().apply {
+ low =
+ Quantity().apply {
+ value = BigDecimal.valueOf(4.8)
+ unit = "kPa"
+ code = "kPa"
+ system = "http:https://unitsofmeasure.org"
+ }
+ high =
+ Quantity().apply {
+ value = BigDecimal.valueOf(6.0)
+ unit = "kPa"
+ code = "kPa"
+ system = "http:https://unitsofmeasure.org"
+ }
+ }
+ )
+ interpretation =
+ listOf(
+ CodeableConcept(
+ Coding(
+ "http:https://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation",
+ "H",
+ "high"
+ )
+ )
+ )
+ }
+
+ interface TestHttpService {
+
+ @GET suspend fun get(@Url path: String): Resource
+
+ @POST(".") suspend fun post(@Body bundle: Bundle): Resource
+
+ class Builder(
+ private val baseUrl: String,
+ private val networkConfiguration: NetworkConfiguration
+ ) {
+
+ fun build(): TestHttpService {
+ val client =
+ OkHttpClient.Builder()
+ .connectTimeout(networkConfiguration.connectionTimeOut, TimeUnit.SECONDS)
+ .readTimeout(networkConfiguration.readTimeOut, TimeUnit.SECONDS)
+ .writeTimeout(networkConfiguration.writeTimeOut, TimeUnit.SECONDS)
+ .apply {
+ if (networkConfiguration.uploadWithGzip) {
+ addInterceptor(GzipUploadInterceptor)
+ }
+ }
+ .build()
+ return Retrofit.Builder()
+ .baseUrl(baseUrl)
+ .client(client)
+ .addConverterFactory(FhirConverterFactory.create())
+ .build()
+ .create(TestHttpService::class.java)
+ }
+ }
+
+ companion object {
+ fun builder(baseUrl: String, networkConfiguration: NetworkConfiguration) =
+ Builder(baseUrl, networkConfiguration)
+ }
+ }
+}

demo/src/main/java/com/google/android/fhir/demo/FhirApplication.kt

@@ -22,6 +22,7 @@ import com.google.android.fhir.DatabaseErrorStrategy.RECREATE_AT_OPEN
 import com.google.android.fhir.FhirEngine
 import com.google.android.fhir.FhirEngineConfiguration
 import com.google.android.fhir.FhirEngineProvider
+import com.google.android.fhir.NetworkConfiguration
 import com.google.android.fhir.ServerConfiguration
 import com.google.android.fhir.datacapture.DataCaptureConfig
 import com.google.android.fhir.datacapture.XFhirQueryResolver
@@ -55,7 +56,8 @@ class FhirApplication : Application(), DataCaptureConfig.Provider {
  HttpLogger.Configuration(
  if (BuildConfig.DEBUG) HttpLogger.Level.BODY else HttpLogger.Level.BASIC
  )
- ) { Timber.tag("App-HttpLog").d(it) }
+ ) { Timber.tag("App-HttpLog").d(it) },
+ networkConfiguration = NetworkConfiguration(uploadWithGzip = false)
  )
  )
  )

engine/src/main/java/com/google/android/fhir/FhirEngineProvider.kt

@@ -161,5 +161,7 @@ data class NetworkConfiguration(
  /** Read timeout (in seconds) for network connection. The default is 10 seconds. */
  val readTimeOut: Long = 10,
  /** Write timeout (in seconds) for network connection. The default is 10 seconds. */
- val writeTimeOut: Long = 10
+ val writeTimeOut: Long = 10,
+ /** Compresses requests when uploading to a server that supports gzip. */
+ val uploadWithGzip: Boolean = false,
 )

engine/src/main/java/com/google/android/fhir/sync/remote/FhirConverter.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,8 +28,7 @@ import org.hl7.fhir.r4.model.Resource
 import retrofit2.Converter
 import retrofit2.Retrofit
 
-internal class FhirConverterFactory private constructor(val fhirContext: FhirContext) :
- Converter.Factory() {
+class FhirConverterFactory private constructor(val fhirContext: FhirContext) : Converter.Factory() {
  override fun responseBodyConverter(
  type: Type,
  annotations: Array<Annotation>,

engine/src/main/java/com/google/android/fhir/sync/remote/GzipUploadInterceptor.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.fhir.sync.remote
+
+import okhttp3.Interceptor
+import okhttp3.MediaType
+import okhttp3.RequestBody
+import okhttp3.Response
+import okio.Buffer
+import okio.BufferedSink
+import okio.GzipSink
+import okio.buffer
+
+const val CONTENT_ENCODING_HEADER_NAME = "Content-Encoding"
+
+/** Compresses upload requests with gzip. */
+object GzipUploadInterceptor : Interceptor {
+ override fun intercept(chain: Interceptor.Chain): Response {
+ val uncompressedRequest = chain.request()
+ if (uncompressedRequest.body == null) {
+ return chain.proceed(uncompressedRequest)
+ }
+
+ val encodingHeader =
+ if (uncompressedRequest.header(CONTENT_ENCODING_HEADER_NAME) != null) {
+ "${uncompressedRequest.header(CONTENT_ENCODING_HEADER_NAME)}, gzip"
+ } else {
+ "gzip"
+ }
+
+ val compressedRequest =
+ uncompressedRequest
+ .newBuilder()
+ .header(CONTENT_ENCODING_HEADER_NAME, encodingHeader)
+ .method(uncompressedRequest.method, addContentLength(gzip(uncompressedRequest.body!!)))
+ .build()
+
+ return chain.proceed(compressedRequest)
+ }
+
+ private fun gzip(body: RequestBody): RequestBody =
+ object : RequestBody() {
+ override fun contentType(): MediaType? = body.contentType()
+
+ override fun writeTo(sink: BufferedSink) {
+ val gzipBufferedSink: BufferedSink = GzipSink(sink).buffer()
+ body.writeTo(gzipBufferedSink)
+ gzipBufferedSink.close()
+ }
+ }
+
+ private fun addContentLength(requestBody: RequestBody): RequestBody {
+ val buffer = Buffer()
+ requestBody.writeTo(buffer)
+ return object : RequestBody() {
+ override fun contentType(): MediaType? = requestBody.contentType()
+
+ override fun contentLength(): Long = buffer.size
+
+ override fun writeTo(sink: BufferedSink) {
+ sink.write(buffer.snapshot())
+ }
+ }
+ }
+}

engine/src/main/java/com/google/android/fhir/sync/remote/RetrofitHttpService.kt

@@ -59,6 +59,9 @@ internal interface RetrofitHttpService : FhirHttpService {
  .readTimeout(networkConfiguration.readTimeOut, TimeUnit.SECONDS)
  .writeTimeout(networkConfiguration.writeTimeOut, TimeUnit.SECONDS)
  .apply {
+ if (networkConfiguration.uploadWithGzip) {
+ addInterceptor(GzipUploadInterceptor)
+ }
  httpLoggingInterceptor?.let { addInterceptor(it) }
  authenticator?.let {
  addInterceptor(