add OpenCV YUV rotate

app/src/main/java/com/taobao/android/mnn/MNNNetNative.java

@@ -19,9 +19,10 @@ static void loadGpuLibrary(String name) {
 
  static {
  System.loadLibrary("MNN");
- loadGpuLibrary("MNN_Vulkan");
- loadGpuLibrary("MNN_OpenCL");
- loadGpuLibrary("MNN_GL");
+ // don't need them in this demo
+// loadGpuLibrary("MNN_Vulkan");
+// loadGpuLibrary("MNN_OpenCL");
+// loadGpuLibrary("MNN_GL");
  System.loadLibrary("mnncore");
  }
 

app/src/main/java/ru/gordinmitya/yuv2buf_demo/CompositeConverter.kt

@@ -6,10 +6,10 @@ import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageProxy
 import androidx.lifecycle.LifecycleObserver
 
-class ConversionResult(val method: String, val image: Bitmap, val time: Long)
+class ConversionResult(val method: String, val image: Bitmap, val colorTime: Long, val rotateTime: Long)
 interface ImageConverter {
  fun getName(): String
- fun convert(image: ImageProxy): Pair<Bitmap, Long>
+ fun convert(image: ImageProxy): ConversionResult
 }
 
 class CompositeConverter(
@@ -22,8 +22,7 @@ class CompositeConverter(
  override fun analyze(image: ImageProxy) {
  val results = functions.map { converter ->
  image.planes.forEach { plane -> plane.buffer.rewind() }
- val (bitmap, time) = converter.convert(image)
- ConversionResult(converter.getName(), bitmap, time)
+ converter.convert(image)
  }
  handler.post {
  val size =

app/src/main/java/ru/gordinmitya/yuv2buf_demo/MNNConverter.kt

@@ -34,7 +34,7 @@ class MNNConverter(context: Context) : ImageConverter {
  outputTensor = session.getOutput(null)
  }
 
- override fun convert(image: ImageProxy): Pair<Bitmap, Long> {
+ override fun convert(image: ImageProxy): ConversionResult {
  val tik = System.currentTimeMillis()
 
  val converted = Yuv.toBuffer(image, reuseBuffer)
@@ -87,6 +87,6 @@ class MNNConverter(context: Context) : ImageConverter {
  // we'll do it in CompositeConverter
  // image.close()
 
- return bitmap to tok - tik
+ return ConversionResult(getName(), bitmap, tok - tik, 0)
  }
 }

app/src/main/java/ru/gordinmitya/yuv2buf_demo/MainActivity.kt

@@ -9,6 +9,7 @@ import android.os.Bundle
 import android.os.Handler
 import android.text.method.LinkMovementMethod
 import android.text.util.Linkify
+import android.util.Size
 import android.view.Menu
 import android.view.MenuItem
 import android.view.View
@@ -34,7 +35,7 @@ class MainActivity : AppCompatActivity(), CompositeConverter.Listener {
  private lateinit var analysisExecutor: ExecutorService
  private lateinit var cameraProviderFuture: ListenableFuture<ProcessCameraProvider>
  private lateinit var imageAnalyzer: CompositeConverter
- private lateinit var resultAverages: Array<MovingAverage>
+ private lateinit var resultAverages: Array<Pair<MovingAverage, MovingAverage>>
  private lateinit var resultViews: Array<View>
 
  override fun onCreate(savedInstanceState: Bundle?) {
@@ -47,11 +48,14 @@ class MainActivity : AppCompatActivity(), CompositeConverter.Listener {
 
  analysisExecutor = Executors.newSingleThreadExecutor()
  val converters = arrayOf(
- MNNConverter(this),
  OpenCVConverter(),
- RenderScriptConverter(this)
+ OpenCVRoteterter(),
+ RenderScriptConverter(this),
+ MNNConverter(this)
  )
- resultAverages = Array(converters.size) { MovingAverage(movingAverageSize) }
+ resultAverages = Array(converters.size) {
+ MovingAverage(movingAverageSize) to MovingAverage(movingAverageSize)
+ }
  resultViews = Array(converters.size) {
  return@Array layoutInflater.inflate(R.layout.item_converted, list_results, false)
  }
@@ -69,15 +73,22 @@ class MainActivity : AppCompatActivity(), CompositeConverter.Listener {
 
  @SuppressLint("SetTextI18n")
  override fun onAnalyzed(size: Pair<Int, Int>, results: List<ConversionResult>) {
+ fun Double.format() = String.format("%.2f", this)
+
  if (!lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)) return
  text_size.text = "${size.first}x${size.second}"
  results.forEachIndexed { index, result ->
  val average = resultAverages[index]
- average.add(result.time)
+ average.first.add(result.colorTime)
+ average.second.add(result.rotateTime)
  resultViews[index].let {
  it.image.setImageBitmap(result.image)
  it.text_name.text = result.method
- it.text_time.text = "${result.time}ms\n$average"
+ it.text_time.text = "clr ${result.colorTime}ms\n" +
+ "avg clr ${average.first.avg().format()}\n" +
+ "rot ${result.rotateTime}ms\n" +
+ "avg rot ${average.second.avg().format()}\n" +
+ "total ${(average.first.avg() + average.second.avg()).format()}"
  }
  }
  }
@@ -89,6 +100,7 @@ class MainActivity : AppCompatActivity(), CompositeConverter.Listener {
  .requireLensFacing(CameraSelector.LENS_FACING_BACK)
  .build()
  val analysis = ImageAnalysis.Builder()
+ .setTargetResolution(Size(720, 1280))
  .setTargetRotation(preview_view.display.rotation)
  .build()
  .also {

app/src/main/java/ru/gordinmitya/yuv2buf_demo/OpenCVConverter.kt

@@ -12,11 +12,11 @@ import java.nio.ByteBuffer
 
 
 class OpenCVConverter() : ImageConverter {
- override fun getName(): String = "OpenCV"
+ override fun getName(): String = "OpenCV (RGB rotate)"
 
  private var reuseBuffer: ByteBuffer? = null
 
- override fun convert(image: ImageProxy): Pair<Bitmap, Long> {
+ override fun convert(image: ImageProxy): ConversionResult {
  val tik = System.currentTimeMillis()
 
  val converted = Yuv.toBuffer(image, reuseBuffer)
@@ -31,12 +31,14 @@ class OpenCVConverter() : ImageConverter {
  val rgbMat = Mat(image.height, image.width, CvType.CV_8UC4)
  Imgproc.cvtColor(yuvMat, rgbMat, format)
 
- val tok = System.currentTimeMillis()
+ val tokColor = System.currentTimeMillis()
 
  if (image.imageInfo.rotationDegrees != 0) {
  Core.rotate(rgbMat, rgbMat, image.imageInfo.rotationDegrees / 90 - 1)
  }
 
+ val tokRotate = System.currentTimeMillis()
+
  val bitmap = Bitmap.createBitmap(rgbMat.cols(), rgbMat.rows(), Bitmap.Config.ARGB_8888)
  Utils.matToBitmap(rgbMat, bitmap)
 
@@ -46,6 +48,6 @@ class OpenCVConverter() : ImageConverter {
  // we'll do it in CompositeConverter
  // image.close()
 
- return bitmap to tok - tik
+ return ConversionResult(getName(), bitmap, tokColor - tik, tokRotate - tokColor)
  }
 }

app/src/main/java/ru/gordinmitya/yuv2buf_demo/OpenCVRoteterter.kt

@@ -0,0 +1,77 @@
+package ru.gordinmitya.yuv2buf_demo
+
+import android.graphics.Bitmap
+import androidx.camera.core.ImageProxy
+import org.opencv.android.Utils
+import org.opencv.core.Core
+import org.opencv.core.CvType
+import org.opencv.core.Mat
+import org.opencv.imgproc.Imgproc
+import ru.gordinmitya.yuv2buf.Yuv
+import java.nio.ByteBuffer
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+
+
+// Here I try to rotate image in YUV format because it's 2x smaller than RGB
+class OpenCVRoteterter() : ImageConverter {
+ override fun getName(): String = "OpenCV (YUV rotate)"
+
+ private var reuseBuffer: ByteBuffer? = null
+
+ override fun convert(image: ImageProxy): ConversionResult {
+ val tik = System.currentTimeMillis()
+
+ val converted = Yuv.toBuffer(image, reuseBuffer)
+ reuseBuffer = converted.buffer
+
+ val format = when (converted.type) {
+ Yuv.Type.YUV_I420 -> Imgproc.COLOR_YUV2RGB_I420
+ Yuv.Type.YUV_NV21 -> Imgproc.COLOR_YUV2RGB_NV21
+ }
+
+ val ySize = image.width * image.height
+ val cSize = ySize / 4
+
+ val yBuf = converted.buffer
+ val uvBuf = converted.buffer.clipBuffer(ySize, cSize + cSize)
+
+ val yMat = Mat(image.height, image.width, CvType.CV_8UC1, yBuf)
+ val uvMat = Mat(image.height / 2, image.width / 2, CvType.CV_8UC2, uvBuf)
+
+ val tRotateStart = System.currentTimeMillis()
+ if (image.imageInfo.rotationDegrees != 0) {
+ Core.rotate(yMat, yMat, image.imageInfo.rotationDegrees / 90 - 1)
+ Core.rotate(uvMat, uvMat, image.imageInfo.rotationDegrees / 90 - 1)
+ }
+ val tRotateEnd = System.currentTimeMillis()
+
+ val rgbMat = Mat(image.height, image.width, CvType.CV_8UC4)
+
+ Imgproc.cvtColorTwoPlane(yMat, uvMat, rgbMat, format)
+
+ val tok = System.currentTimeMillis()
+
+ val bitmap = Bitmap.createBitmap(rgbMat.cols(), rgbMat.rows(), Bitmap.Config.ARGB_8888)
+ Utils.matToBitmap(rgbMat, bitmap)
+
+ // don't forget to call image.close() here
+ // but as long as we have many converters
+ // we'll do it in CompositeConverter
+ // image.close()
+
+ val colorTime = tRotateStart - tik + tok - tRotateEnd
+
+ return ConversionResult(getName(), bitmap, colorTime, tRotateEnd - tRotateStart)
+ }
+
+ private fun ByteBuffer.clipBuffer(
+ start: Int,
+ size: Int
+ ): ByteBuffer? {
+ val duplicate = this.duplicate()
+ duplicate.position(start)
+ duplicate.limit(start + size)
+ return duplicate.slice()
+ }
+}

app/src/main/java/ru/gordinmitya/yuv2buf_demo/RenderScriptConverter.kt

@@ -22,7 +22,7 @@ class RenderScriptConverter(context: Context) : ImageConverter {
 
  override fun getName(): String = "RenderScript"
 
- override fun convert(image: ImageProxy): Pair<Bitmap, Long> {
+ override fun convert(image: ImageProxy): ConversionResult {
  val tik = System.currentTimeMillis()
 
  val converted = Yuv.toBuffer(image, reuseBuffer)
@@ -56,7 +56,7 @@ class RenderScriptConverter(context: Context) : ImageConverter {
 
  intrinsic.forEach(out)
 
- val tok = System.currentTimeMillis()
+ val tokColor = System.currentTimeMillis()
 
  var bitmap = Bitmap.createBitmap(image.width, image.height, Bitmap.Config.ARGB_8888)
  out!!.copyTo(bitmap)
@@ -75,11 +75,14 @@ class RenderScriptConverter(context: Context) : ImageConverter {
  )
  }
 
+ val tokRotate = System.currentTimeMillis()
+
+
  // don't forget to call image.close() here
  // but as long as we have many converters
  // we'll do it in CompositeConverter
  // image.close()
 
- return bitmap to tok - tik
+ return ConversionResult(getName(), bitmap, tokColor - tik, tokRotate - tokColor)
  }
 }