benchmark comparison with MLKit solution

yuv2buf-shared-test/src/main/java/ru/gordinmitya/yuv2buf/YuvCommon.java

@@ -50,6 +50,68 @@ static Yuv.ImageWrapper make(@Yuv.YuvType int type, int width, int height, int r
  return new Yuv.ImageWrapper(width, height, y, u, v);
  }
 
+ static boolean checkOutput(@Yuv.YuvType int type, ByteBuffer buffer, int width, int height) {
+ if (ImageFormat.YUV_420_888 == type) {
+ return checkYuv420(buffer, width, height);
+ } else if (ImageFormat.NV21 == type) {
+ return checkNV21(buffer, width, height);
+ }
+ throw new IllegalArgumentException("Unknown type " + type);
+ }
+
+ static boolean checkYuv420(ByteBuffer buffer, int width, int height) {
+ byte[] array = buffer2array(buffer);
+
+ int sizeY = width * height;
+ int sizeChroma = sizeY / 4;
+ int sizeTotal = sizeY + sizeChroma * 2;
+
+ for (int i = 0; i < sizeY; i++) {
+ if (array[i] != Y) {
+ System.out.println("Missmatch at " + i + " expected " + Y + " but was " + array[i]);
+ return false;
+ }
+ }
+ for (int i = sizeY; i < sizeY + sizeChroma; i++) {
+ if (array[i] != U) {
+ System.out.println("Missmatch at " + i + " expected " + U + " but was " + array[i]);
+ return false;
+ }
+ }
+ for (int i = sizeY + sizeChroma; i < sizeTotal; i++) {
+ if (array[i] != V) {
+ System.out.println("Missmatch at " + i + " expected " + V + " but was " + array[i]);
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ static boolean checkNV21(ByteBuffer buffer, int width, int height) {
+ byte[] array = buffer2array(buffer);
+
+ int sizeY = width * height;
+ int sizeChroma = sizeY / 4;
+ int sizeTotal = sizeY + sizeChroma * 2;
+
+ for (int i = 0; i < sizeY; i++) {
+ if (array[i] != Y) {
+ System.out.println("Missmatch at " + i + " expected " + Y + " but was " + array[i]);
+ return false;
+ }
+ }
+ for (int i = sizeY; i < sizeTotal; i++) {
+ byte expected = i % 2 == 0 ? V : U;
+ if (array[i] != expected) {
+ System.out.println("Missmatch at " + i + " expected " + expected + " but was " + array[i]);
+ return false;
+ }
+ }
+
+ return true;
+ }
+
  static byte[] buffer2array(ByteBuffer buffer) {
  byte[] array = new byte[buffer.capacity()];
  ByteBuffer copy = buffer.duplicate();

yuv2buf/src/androidTest/java/ru/gordinmitya/yuv2buf/InstrumentedBenchmark.kt

@@ -0,0 +1,112 @@
+package ru.gordinmitya.yuv2buf
+
+import android.graphics.ImageFormat
+import android.os.SystemClock
+import android.util.Log
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import ru.gordinmitya.yuv2buf.MLKit_BitmapUtils.Image_Plane
+
+inline fun <R> measure(block: () -> R): Pair<R, Long> {
+ val tik = SystemClock.elapsedRealtimeNanos()
+ val result = block()
+ val tok = SystemClock.elapsedRealtimeNanos()
+ return Pair(result, tok - tik)
+}
+
+@RunWith(AndroidJUnit4::class)
+class InstrumentedBenchmark {
+ private val iterations = 100
+
+ private val width = 1280
+ private val height = 720
+ private val additionalRowStride = 1536
+
+ private fun baseline(image: Yuv.ImageWrapper): Double {
+ val reuseBuffer = Yuv.toBuffer(image, null).buffer
+ val timings = ArrayList<Long>(iterations)
+ repeat(iterations) { _ ->
+ val (converted, time) = measure {
+ Yuv.toBuffer(image, reuseBuffer)
+ }
+ assertTrue(YuvCommon.checkOutput(converted.type, converted.buffer, width, height))
+ timings.add(time)
+ }
+ return timings.average()
+ }
+
+ @Test
+ fun baseline_nv21_noRowStride() {
+ val image = YuvCommon.make(ImageFormat.NV21, width, height, width)
+ val avg = baseline(image)
+ Log.d("baseline_nv21_noRowStride", "$avg ns")
+ }
+
+ @Test
+ fun baseline_nv21_withRowStride() {
+ val image = YuvCommon.make(ImageFormat.NV21, width, height, additionalRowStride)
+ val avg = baseline(image)
+ Log.d("baseline_nv21_withRowStride", "$avg ns")
+ }
+
+ @Test
+ fun baseline_yuv420_noRowStride() {
+ val image = YuvCommon.make(ImageFormat.YUV_420_888, width, height, width)
+ val avg = baseline(image)
+ Log.d("baseline_yuv420_noRowStride", "$avg ns")
+ }
+
+ @Test
+ fun baseline_yuv420_withRowStride() {
+ val image = YuvCommon.make(ImageFormat.YUV_420_888, width, height, additionalRowStride)
+ val avg = baseline(image)
+ Log.d("baseline_yuv420_withRowStride", "$avg ns")
+ }
+
+ private fun mlkit(image: Yuv.ImageWrapper): Double {
+ val timings = ArrayList<Long>(iterations)
+ val planes = arrayOf(
+ Image_Plane(image.y.buffer, image.y.rowStride, image.y.pixelStride),
+ Image_Plane(image.u.buffer, image.u.rowStride, image.u.pixelStride),
+ Image_Plane(image.v.buffer, image.v.rowStride, image.v.pixelStride),
+ )
+ repeat(iterations) { _ ->
+ val (buffer, time) = measure {
+ MLKit_BitmapUtils.yuv420ThreePlanesToNV21(planes, image.width, image.height)
+ }
+ assertTrue(YuvCommon.checkOutput(ImageFormat.NV21, buffer, width, height))
+ timings.add(time)
+ }
+ return timings.average()
+ }
+
+ @Test
+ fun mlkit_nv21_noRowStride() {
+ val image = YuvCommon.make(ImageFormat.NV21, width, height, width)
+ val avg = mlkit(image)
+ Log.d("mlkit_nv21_noRowStride", "$avg ns")
+ }
+
+ @Test
+ fun mlkit_nv21_withRowStride() {
+ val image = YuvCommon.make(ImageFormat.NV21, width, height, additionalRowStride)
+ val avg = mlkit(image)
+ Log.d("mlkit_nv21_withRowStride", "$avg ns")
+ }
+
+ @Test
+ fun mlkit_yuv420_noRowStride() {
+ val image = YuvCommon.make(ImageFormat.YUV_420_888, width, height, width)
+ val avg = mlkit(image)
+ Log.d("mlkit_yuv420_noRowStride", "$avg ns")
+ }
+
+ @Test
+ fun mlkit_yuv420_withRowStride() {
+ val image = YuvCommon.make(ImageFormat.YUV_420_888, width, height, additionalRowStride)
+ val avg = mlkit(image)
+ Log.d("mlkit_yuv420_withRowStride", "$avg ns")
+ }
+}

yuv2buf/src/androidTest/java/ru/gordinmitya/yuv2buf/MLKit_BitmapUtils.java

@@ -0,0 +1,141 @@
+package ru.gordinmitya.yuv2buf;
+
+// https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java#L177-L220
+
+import android.media.Image;
+
+import java.nio.ByteBuffer;
+
+public class MLKit_BitmapUtils {
+ static class Image_Plane {
+ private final ByteBuffer buffer;
+ private final int rowStride;
+ private final int pixelStride;
+
+ Image_Plane(ByteBuffer buffer, int rowStride, int pixelStride) {
+ this.buffer = buffer;
+ this.rowStride = rowStride;
+ this.pixelStride = pixelStride;
+ }
+
+ public int getRowStride() {
+ return rowStride;
+ }
+
+ public int getPixelStride() {
+ return pixelStride;
+ }
+
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+ }
+
+ /**
+ * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format.
+ */
+ private static boolean areUVPlanesNV21(Image_Plane[] planes, int width, int height) {
+ int imageSize = width * height;
+
+ ByteBuffer uBuffer = planes[1].getBuffer();
+ ByteBuffer vBuffer = planes[2].getBuffer();
+
+ // Backup buffer properties.
+ int vBufferPosition = vBuffer.position();
+ int uBufferLimit = uBuffer.limit();
+
+ // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+ vBuffer.position(vBufferPosition + 1);
+ // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+ uBuffer.limit(uBufferLimit - 1);
+
+ // Check that the buffers are equal and have the expected number of elements.
+ boolean areNV21 =
+ (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+ // Restore buffers to their initial state.
+ vBuffer.position(vBufferPosition);
+ uBuffer.limit(uBufferLimit);
+
+ return areNV21;
+ }
+
+ /**
+ * Unpack an image plane into a byte array.
+ *
+ * <p>The input plane data will be copied in 'out', starting at 'offset' and every pixel will be
+ * spaced by 'pixelStride'. Note that there is no row padding on the output.
+ */
+ private static void unpackPlane(
+ Image_Plane plane, int width, int height, byte[] out, int offset, int pixelStride) {
+ ByteBuffer buffer = plane.getBuffer();
+ buffer.rewind();
+
+ // Compute the size of the current plane.
+ // We assume that it has the aspect ratio as the original image.
+ int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride();
+ if (numRow == 0) {
+ return;
+ }
+ int scaleFactor = height / numRow;
+ int numCol = width / scaleFactor;
+
+ // Extract the data in the output buffer.
+ int outputPos = offset;
+ int rowStart = 0;
+ for (int row = 0; row < numRow; row++) {
+ int inputPos = rowStart;
+ for (int col = 0; col < numCol; col++) {
+ out[outputPos] = buffer.get(inputPos);
+ outputPos += pixelStride;
+ inputPos += plane.getPixelStride();
+ }
+ rowStart += plane.getRowStride();
+ }
+ }
+
+ /**
+ * Converts YUV_420_888 to NV21 bytebuffer.
+ *
+ * <p>The NV21 format consists of a single byte array containing the Y, U and V values. For an
+ * image of size S, the first S positions of the array contain all the Y values. The remaining
+ * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both
+ * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain
+ * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU
+ *
+ * <p>YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled
+ * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and
+ * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into
+ * the first part of the NV21 array. The U and V planes may already have the representation in the
+ * NV21 format. This happens if the planes share the same buffer, the V buffer is one position
+ * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy
+ * them to the NV21 array.
+ */
+ public static ByteBuffer yuv420ThreePlanesToNV21(
+ Image_Plane[] yuv420888planes, int width, int height) {
+ int imageSize = width * height;
+ byte[] out = new byte[imageSize + 2 * (imageSize / 4)];
+
+ if (areUVPlanesNV21(yuv420888planes, width, height)) {
+ // Copy the Y values.
+ yuv420888planes[0].getBuffer().get(out, 0, imageSize);
+
+ ByteBuffer uBuffer = yuv420888planes[1].getBuffer();
+ ByteBuffer vBuffer = yuv420888planes[2].getBuffer();
+ // Get the first V value from the V buffer, since the U buffer does not contain it.
+ vBuffer.get(out, imageSize, 1);
+ // Copy the first U value and the remaining VU values from the U buffer.
+ uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1);
+ } else {
+ // Fallback to copying the UV values one by one, which is slower but also works.
+ // Unpack Y.
+ unpackPlane(yuv420888planes[0], width, height, out, 0, 1);
+ // Unpack U.
+ unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2);
+ // Unpack V.
+ unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2);
+ }
+
+ return ByteBuffer.wrap(out);
+ }
+}

yuv2buf/src/test/java/ru/gordinmitya/yuv2buf/Yuv420Test.java

@@ -10,10 +10,7 @@
 import java.util.Collection;
 
 import static org.junit.Assert.assertEquals;
-import static ru.gordinmitya.yuv2buf.YuvCommon.U;
-import static ru.gordinmitya.yuv2buf.YuvCommon.V;
-import static ru.gordinmitya.yuv2buf.YuvCommon.Y;
-import static ru.gordinmitya.yuv2buf.YuvCommon.buffer2array;
+import static org.junit.Assert.assertTrue;
 
 
 @RunWith(Parameterized.class)
@@ -45,15 +42,6 @@ public void check() {
  int sizeChroma = sizeY / 4;
  int sizeTotal = sizeY + sizeChroma * 2;
  assertEquals(sizeTotal, buffer.capacity());
- byte[] array = buffer2array(converted.buffer);
-
- for (int i = 0; i < sizeY; i++)
- assertEquals(Y, array[i]);
-
- for (int i = sizeY; i < sizeY + sizeChroma; i++)
- assertEquals(U, array[i]);
-
- for (int i = sizeY + sizeChroma; i < sizeTotal; i++)
- assertEquals(V, array[i]);
+ assertTrue(YuvCommon.checkYuv420(converted.buffer, width, height));
  }
 }