perf(ext/ffi): use fast api calls for 64bit return types (denoland#15313

)

ext/ffi/00_ffi.js

@@ -7,9 +7,17 @@
  const {
  BigInt,
  ObjectDefineProperty,
+ ArrayPrototypeMap,
+ Number,
+ NumberIsSafeInteger,
+ ArrayPrototypeJoin,
  ObjectPrototypeIsPrototypeOf,
  PromisePrototypeThen,
  TypeError,
+ Int32Array,
+ Uint32Array,
+ BigInt64Array,
+ Function,
  } = window.__bootstrap.primordials;
 
  function unpackU64(returnValue) {
@@ -209,6 +217,10 @@
  type === "usize" || type === "isize";
  }
 
+ function isI64(type) {
+ return type === "i64" || type === "isize";
+ }
+
  class UnsafeCallback {
  #refcount;
  #rid;
@@ -293,11 +305,11 @@
  );
  continue;
  }
+ const resultType = symbols[symbol].result;
+ const needsUnpacking = isReturnedAsBigInt(resultType);
 
  const isNonBlocking = symbols[symbol].nonblocking;
  if (isNonBlocking) {
- const resultType = symbols[symbol].result;
- const needsUnpacking = isReturnedAsBigInt(resultType);
  ObjectDefineProperty(
  this.symbols,
  symbol,
@@ -326,6 +338,38 @@
  },
  );
  }
+
+ if (needsUnpacking && !isNonBlocking) {
+ const call = this.symbols[symbol];
+ const parameters = symbols[symbol].parameters;
+ const vi = new Int32Array(2);
+ const vui = new Uint32Array(vi.buffer);
+ const b = new BigInt64Array(vi.buffer);
+
+ const params = ArrayPrototypeJoin(
+ ArrayPrototypeMap(parameters, (_, index) => `p${index}`),
+ ", ",
+ );
+ // Make sure V8 has no excuse to not optimize this function.
+ this.symbols[symbol] = new Function(
+ "vi",
+ "vui",
+ "b",
+ "call",
+ "NumberIsSafeInteger",
+ "Number",
+ `return function (${params}) {
+ call(${params}${parameters.length > 0 ? ", " : ""}vi);
+ ${
+ isI64(resultType)
+ ? `const n1 = Number(b[0])`
+ : `const n1 = vui[0] + 2 ** 32 * vui[1]` // Faster path for u64
+ };
+ if (NumberIsSafeInteger(n1)) return n1;
+ return b[0];
+ }`,
+ )(vi, vui, b, call, NumberIsSafeInteger, Number);
+ }
  }
  }
 

ext/ffi/jit_trampoline.rs

@@ -58,12 +58,15 @@ fn native_to_c(ty: &NativeType) -> &'static str {
 
 pub(crate) fn codegen(sym: &crate::Symbol) -> String {
  let mut c = String::from(include_str!("prelude.h"));
- let ret = native_to_c(&sym.result_type);
+ let needs_unwrap = crate::needs_unwrap(sym.result_type);
+
+ // Return type of the FFI call.
+ let ffi_ret = native_to_c(&sym.result_type);
+ // Return type of the trampoline.
+ let ret = if needs_unwrap { "void" } else { ffi_ret };
 
  // extern <return_type> func(
- c += "\nextern ";
- c += ret;
- c += " func(";
+ let _ = write!(c, "\nextern {ffi_ret} func(");
  // <param_type> p0, <param_type> p1, ...);
  for (i, ty) in sym.parameter_types.iter().enumerate() {
  if i > 0 {
@@ -83,20 +86,35 @@ pub(crate) fn codegen(sym: &crate::Symbol) -> String {
  c += native_arg_to_c(ty);
  let _ = write!(c, " p{i}");
  }
+ if needs_unwrap {
+ let _ = write!(c, ", struct FastApiTypedArray* const p_ret");
+ }
  c += ") {\n";
- // return func(p0, p1, ...);
- c += " return func(";
- for (i, ty) in sym.parameter_types.iter().enumerate() {
- if i > 0 {
- c += ", ";
- }
- if matches!(ty, NativeType::Pointer) {
- let _ = write!(c, "p{i}->data");
- } else {
- let _ = write!(c, "p{i}");
+ // func(p0, p1, ...);
+ let mut call_s = String::from("func(");
+ {
+ for (i, ty) in sym.parameter_types.iter().enumerate() {
+ if i > 0 {
+ call_s += ", ";
+ }
+ if matches!(ty, NativeType::Pointer) {
+ let _ = write!(call_s, "p{i}->data");
+ } else {
+ let _ = write!(call_s, "p{i}");
+ }
  }
+ call_s += ");\n";
  }
- c += ");\n}\n\n";
+ if needs_unwrap {
+ // <return_type> r = func(p0, p1, ...);
+ // ((<return_type>*)p_ret->data)[0] = r;
+ let _ = write!(c, " {ffi_ret} r = {call_s}");
+ let _ = writeln!(c, " (({ffi_ret}*)p_ret->data)[0] = r;");
+ } else {
+ // return func(p0, p1, ...);
+ let _ = write!(c, " return {call_s}");
+ }
+ c += "}\n\n";
  c
 }
 
@@ -190,6 +208,22 @@ mod tests {
  \n return func(p0->data, p1->data);\n\
  }\n\n",
  );
+ assert_codegen(
+ codegen(vec![], NativeType::U64),
+ "extern uint64_t func();\n\n\
+ void func_trampoline(void* recv, struct FastApiTypedArray* const p_ret) {\
+ \n uint64_t r = func();\
+ \n ((uint64_t*)p_ret->data)[0] = r;\n\
+ }\n\n",
+ );
+ assert_codegen(
+ codegen(vec![NativeType::Pointer, NativeType::Pointer], NativeType::U64),
+ "extern uint64_t func(void* p0, void* p1);\n\n\
+ void func_trampoline(void* recv, struct FastApiTypedArray* p0, struct FastApiTypedArray* p1, struct FastApiTypedArray* const p_ret) {\
+ \n uint64_t r = func(p0->data, p1->data);\
+ \n ((uint64_t*)p_ret->data)[0] = r;\n\
+ }\n\n",
+ );
  }
 
  #[test]

ext/ffi/lib.rs

@@ -752,9 +752,8 @@ impl From<&NativeType> for fast_api::Type {
  }
 }
 
-#[cfg(not(target_os = "windows"))]
-fn is_fast_api_rv(rv: NativeType) -> bool {
- !matches!(
+fn needs_unwrap(rv: NativeType) -> bool {
+ matches!(
  rv,
  NativeType::Function
  | NativeType::Pointer
@@ -765,6 +764,10 @@ fn is_fast_api_rv(rv: NativeType) -> bool {
  )
 }
 
+fn is_i64(rv: NativeType) -> bool {
+ matches!(rv, NativeType::I64 | NativeType::ISize)
+}
+
 // Create a JavaScript function for synchronous FFI call to
 // the given symbol.
 fn make_sync_fn<'s>(
@@ -780,8 +783,12 @@ fn make_sync_fn<'s>(
  #[cfg(not(target_os = "windows"))]
  let mut fast_allocations: Option<*mut ()> = None;
  #[cfg(not(target_os = "windows"))]
- if !sym.can_callback && is_fast_api_rv(sym.result_type) {
- let ret = fast_api::Type::from(&sym.result_type);
+ if !sym.can_callback {
+ let needs_unwrap = needs_unwrap(sym.result_type);
+ let ret = match needs_unwrap {
+ true => fast_api::Type::Void,
+ false => fast_api::Type::from(&sym.result_type),
+ };
 
  let mut args = sym
  .parameter_types
@@ -790,6 +797,9 @@ fn make_sync_fn<'s>(
  .collect::<Vec<_>>();
  // recv
  args.insert(0, fast_api::Type::V8Value);
+ if needs_unwrap {
+ args.push(fast_api::Type::TypedArray(fast_api::CType::Int32));
+ }
  let symbol_trampoline =
  jit_trampoline::gen_trampoline(sym.clone()).expect("gen_trampoline");
  fast_ffi_templ = Some(FfiFastCallTemplate {
@@ -810,11 +820,46 @@ fn make_sync_fn<'s>(
  // SAFETY: The pointer will not be deallocated until the function is
  // garbage collected.
  let symbol = unsafe { &*(external.value() as *const Symbol) };
+ let needs_unwrap = match needs_unwrap(symbol.result_type) {
+ true => Some(args.get(symbol.parameter_types.len() as i32)),
+ false => None,
+ };
  match ffi_call_sync(scope, args, symbol) {
  Ok(result) => {
- // SAFETY: Same return type declared to libffi; trust user to have it right beyond that.
- let result = unsafe { result.to_v8(scope, symbol.result_type) };
- rv.set(result.v8_value);
+ match needs_unwrap {
+ Some(v) => {
+ let view: v8::Local<v8::ArrayBufferView> = v.try_into().unwrap();
+ let backing_store =
+ view.buffer(scope).unwrap().get_backing_store();
+
+ if is_i64(symbol.result_type) {
+ // SAFETY: v8::SharedRef<v8::BackingStore> is similar to Arc<[u8]>,
+ // it points to a fixed continuous slice of bytes on the heap.
+ let bs = unsafe {
+ &mut *(&backing_store[..] as *const _ as *mut [u8]
+ as *mut i64)
+ };
+ // SAFETY: We already checked that type == I64
+ let value = unsafe { result.i64_value };
+ *bs = value;
+ } else {
+ // SAFETY: v8::SharedRef<v8::BackingStore> is similar to Arc<[u8]>,
+ // it points to a fixed continuous slice of bytes on the heap.
+ let bs = unsafe {
+ &mut *(&backing_store[..] as *const _ as *mut [u8]
+ as *mut u64)
+ };
+ // SAFETY: We checked that type == U64
+ let value = unsafe { result.u64_value };
+ *bs = value;
+ }
+ }
+ None => {
+ // SAFETY: Same return type declared to libffi; trust user to have it right beyond that.
+ let result = unsafe { result.to_v8(scope, symbol.result_type) };
+ rv.set(result.v8_value);
+ }
+ }
  }
  Err(err) => {
  deno_core::_ops::throw_type_error(scope, err.to_string());

test_ffi/src/lib.rs

@@ -404,3 +404,10 @@ pub struct Structure {
 
 #[no_mangle]
 pub static mut static_ptr: Structure = Structure { _data: 42 };
+
+static STRING: &str = "Hello, world!\0";
+
+#[no_mangle]
+extern "C" fn ffi_string() -> *const u8 {
+ STRING.as_ptr()
+}

test_ffi/tests/bench.js

@@ -12,6 +12,8 @@ const libPath = `${targetDir}/${libPrefix}test_ffi.${libSuffix}`;
 const dylib = Deno.dlopen(libPath, {
  "nop": { parameters: [], result: "void" },
  "add_u32": { parameters: ["u32", "u32"], result: "u32" },
+ "add_u64": { parameters: ["u64", "u64"], result: "u64" },
+ "ffi_string": { parameters: [], result: "pointer" },
  "hash": { parameters: ["pointer", "u32"], result: "u32" },
  "nop_u8": { parameters: ["u8"], result: "void" },
  "nop_i8": { parameters: ["i8"], result: "void" },
@@ -227,16 +229,42 @@ Deno.bench("nop()", () => {
  nop();
 });
 
+const bytes = new Uint8Array(64);
+
+const { hash } = dylib.symbols;
+Deno.bench("hash()", () => {
+ hash(bytes, bytes.byteLength);
+});
+
+const { ffi_string } = dylib.symbols;
+Deno.bench(
+ "c string",
+ () => new Deno.UnsafePointerView(ffi_string()).getCString(),
+);
+
 const { add_u32 } = dylib.symbols;
 Deno.bench("add_u32()", () => {
  add_u32(1, 2);
 });
 
-const bytes = new Uint8Array(64);
+const { return_buffer } = dylib.symbols;
+Deno.bench("return_buffer()", () => {
+ return_buffer();
+});
 
-const { hash } = dylib.symbols;
-Deno.bench("hash()", () => {
- hash(bytes, bytes.byteLength);
+const { add_u64 } = dylib.symbols;
+Deno.bench("add_u64()", () => {
+ add_u64(1, 2);
+});
+
+const { return_u64 } = dylib.symbols;
+Deno.bench("return_u64()", () => {
+ return_u64();
+});
+
+const { return_i64 } = dylib.symbols;
+Deno.bench("return_i64()", () => {
+ return_i64();
 });
 
 const { nop_u8 } = dylib.symbols;
@@ -348,16 +376,6 @@ Deno.bench("return_i32()", () => {
  return_i32();
 });
 
-const { return_u64 } = dylib.symbols;
-Deno.bench("return_u64()", () => {
- return_u64();
-});
-
-const { return_i64 } = dylib.symbols;
-Deno.bench("return_i64()", () => {
- return_i64();
-});
-
 const { return_usize } = dylib.symbols;
 Deno.bench("return_usize()", () => {
  return_usize();
@@ -378,11 +396,6 @@ Deno.bench("return_f64()", () => {
  return_f64();
 });
 
-const { return_buffer } = dylib.symbols;
-Deno.bench("return_buffer()", () => {
- return_buffer();
-});
-
 // Nonblocking calls
 
 const { nop_nonblocking } = dylib.symbols;

test_ffi/tests/test.js

@@ -197,7 +197,20 @@ dylib.symbols.print_buffer(buffer, buffer.length);
 const subarray = buffer.subarray(3);
 dylib.symbols.print_buffer(subarray, subarray.length - 2);
 dylib.symbols.print_buffer2(buffer, buffer.length, buffer2, buffer2.length);
-const ptr0 = dylib.symbols.return_buffer();
+
+const { return_buffer } = symbols;
+function returnBuffer() { return return_buffer(); };
+
+%PrepareFunctionForOptimization(returnBuffer);
+returnBuffer();
+%OptimizeFunctionOnNextCall(returnBuffer);
+const ptr0 = returnBuffer();
+
+const status = %GetOptimizationStatus(returnBuffer);
+if (!(status & (1 << 4))) {
+ throw new Error("returnBuffer is not optimized");
+}
+
 dylib.symbols.print_buffer(ptr0, 8);
 const ptrView = new Deno.UnsafePointerView(ptr0);
 const into = new Uint8Array(6);