fix(msgpack): encode huge objects (denoland#3698)

Co-authored-by: Asher Gomez <[email protected]>

bytes/concat.ts

@@ -8,18 +8,43 @@
  * const a = new Uint8Array([0, 1, 2]);
  * const b = new Uint8Array([3, 4, 5]);
  * console.log(concat(a, b)); // [0, 1, 2, 3, 4, 5]
+ * ```
  */
-export function concat(...buf: Uint8Array[]): Uint8Array {
+export function concat(buf: Uint8Array[]): Uint8Array;
+export function concat(...buf: Uint8Array[]): Uint8Array;
+export function concat(...buf: (Uint8Array | Uint8Array[])[]): Uint8Array {
+ // No need to concatenate if there is only one element in array or sub-array
+ if (buf.length === 1) {
+ if (!Array.isArray(buf[0])) {
+ return buf[0];
+ } else if (buf[0].length === 1) {
+ return buf[0][0];
+ }
+ }
+
  let length = 0;
  for (const b of buf) {
- length += b.length;
+ if (Array.isArray(b)) {
+ for (const b1 of b) {
+ length += b1.length;
+ }
+ } else {
+ length += b.length;
+ }
  }
 
  const output = new Uint8Array(length);
  let index = 0;
  for (const b of buf) {
- output.set(b, index);
- index += b.length;
+ if (Array.isArray(b)) {
+ for (const b1 of b) {
+ output.set(b1, index);
+ index += b1.length;
+ }
+ } else {
+ output.set(b, index);
+ index += b.length;
+ }
  }
 
  return output;

bytes/concat_test.ts

@@ -21,7 +21,7 @@ Deno.test("[bytes] concat empty arrays", () => {
  assert(u2 !== joined);
 });
 
-Deno.test("[bytes] concat multiple arrays", () => {
+Deno.test("[bytes] concat multiple Uint8Array", () => {
  const encoder = new TextEncoder();
  const u1 = encoder.encode("Hello ");
  const u2 = encoder.encode("W");
@@ -34,3 +34,49 @@ Deno.test("[bytes] concat multiple arrays", () => {
  assert(u1 !== joined);
  assert(u2 !== joined);
 });
+
+Deno.test("[bytes] concat an array of Uint8Array", () => {
+ const a = [
+ new Uint8Array([0, 1, 2, 3]),
+ new Uint8Array([4, 5, 6]),
+ new Uint8Array([7, 8, 9]),
+ ];
+ const joined = concat(a);
+ const expected = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+ assertEquals(joined, expected);
+});
+
+Deno.test("[bytes] concat multiple arrays of Uint8Array using spread operator", () => {
+ const a = [new Uint8Array([0, 1, 2, 3]), new Uint8Array([4, 5, 6, 7, 8, 9])];
+ const b = [
+ new Uint8Array([10, 11]),
+ new Uint8Array([12, 13]),
+ new Uint8Array([14, 15]),
+ new Uint8Array([16]),
+ new Uint8Array([17, 18, 19]),
+ ];
+ const joined = concat(...a, ...b);
+ const expected = new Uint8Array([
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14,
+ 15,
+ 16,
+ 17,
+ 18,
+ 19,
+ ]);
+ assertEquals(joined, expected);
+});

msgpack/encode.ts

@@ -50,7 +50,7 @@ const encoder = new TextEncoder();
 export function encode(object: ValueType) {
  const byteParts: Uint8Array[] = [];
  encodeSlice(object, byteParts);
- return concat(...byteParts);
+ return concat(byteParts);
 }
 
 function encodeFloat64(num: number) {

msgpack/encode_test.ts

@@ -162,3 +162,24 @@ Deno.test("maps", () => {
  const nestedMap = { "a": -1, "b": 2, "c": "three", "d": null, "e": map1 };
  assertEquals(decode(encode(nestedMap)), nestedMap);
 });
+
+Deno.test("huge array with 100k objects", () => {
+ const bigArray = [];
+ for (let i = 0; i < 100000; i++) {
+ bigArray.push({ a: { i: `${i}` }, i: i });
+ }
+ const bigObject = { a: bigArray };
+
+ assertEquals(decode(encode(bigObject)), bigObject);
+});
+
+Deno.test("huge object with 100k properties", () => {
+ const bigObject = {};
+ for (let i = 0; i < 100000; i++) {
+ const _ = Object.defineProperty(bigObject, `prop_${i}`, {
+ value: i,
+ enumerable: true,
+ });
+ }
+ assertEquals(decode(encode(bigObject)), bigObject);
+});