feat(ext/ffi): add support for buffer arguments (denoland#12335)

This commit adds support for passing buffer arguments across 
FFI boundary.


Co-authored-by: eliassjogreen <[email protected]>
Co-authored-by: Bert Belder <[email protected]>

ext/ffi/00_ffi.js

@@ -4,7 +4,9 @@
 ((window) => {
  const core = window.Deno.core;
  const __bootstrap = window.__bootstrap;
-
+ const {
+ ArrayBuffer,
+ } = window.__bootstrap.primordials;
  class DynamicLibrary {
  #rid;
  symbols = {};
@@ -13,15 +15,40 @@
  this.#rid = core.opSync("op_ffi_load", { path, symbols });
 
  for (const symbol in symbols) {
- this.symbols[symbol] = symbols[symbol].nonblocking
- ? (...parameters) =>
- core.opAsync("op_ffi_call_nonblocking", {
+ const isNonBlocking = symbols[symbol].nonblocking;
+
+ this.symbols[symbol] = (...args) => {
+ const parameters = [];
+ const buffers = [];
+
+ for (const arg of args) {
+ if (
+ arg?.buffer instanceof ArrayBuffer &&
+ arg.byteLength !== undefined
+ ) {
+ parameters.push(buffers.length);
+ buffers.push(arg);
+ } else {
+ parameters.push(arg);
+ }
+ }
+
+ if (isNonBlocking) {
+ return core.opAsync("op_ffi_call_nonblocking", {
+ rid: this.#rid,
+ symbol,
+ parameters,
+ buffers,
+ });
+ } else {
+ return core.opSync("op_ffi_call", {
  rid: this.#rid,
  symbol,
  parameters,
- })
- : (...parameters) =>
- core.opSync("op_ffi_call", { rid: this.#rid, symbol, parameters });
+ buffers,
+ });
+ }
+ };
  }
  }
 

ext/ffi/lib.rs

@@ -11,6 +11,7 @@ use deno_core::Extension;
 use deno_core::OpState;
 use deno_core::Resource;
 use deno_core::ResourceId;
+use deno_core::ZeroCopyBuf;
 use dlopen::raw::Library;
 use libffi::middle::Arg;
 use serde::Deserialize;
@@ -131,6 +132,7 @@ enum NativeType {
  ISize,
  F32,
  F64,
+ Buffer,
 }
 
 impl From<NativeType> for libffi::middle::Type {
@@ -149,6 +151,7 @@ impl From<NativeType> for libffi::middle::Type {
  NativeType::ISize => libffi::middle::Type::isize(),
  NativeType::F32 => libffi::middle::Type::f32(),
  NativeType::F64 => libffi::middle::Type::f64(),
+ NativeType::Buffer => libffi::middle::Type::pointer(),
  }
  }
 }
@@ -168,6 +171,7 @@ union NativeValue {
  isize_value: isize,
  f32_value: f32,
  f64_value: f64,
+ buffer: *const u8,
 }
 
 impl NativeValue {
@@ -210,9 +214,14 @@ impl NativeValue {
  NativeType::F64 => Self {
  f64_value: value_as_f64(value),
  },
+ NativeType::Buffer => unreachable!(),
  }
  }
 
+ fn buffer(ptr: *const u8) -> Self {
+ Self { buffer: ptr }
+ }
+
  unsafe fn as_arg(&self, native_type: NativeType) -> Arg {
  match native_type {
  NativeType::Void => Arg::new(&self.void_value),
@@ -228,6 +237,7 @@ impl NativeValue {
  NativeType::ISize => Arg::new(&self.isize_value),
  NativeType::F32 => Arg::new(&self.f32_value),
  NativeType::F64 => Arg::new(&self.f64_value),
+ NativeType::Buffer => Arg::new(&self.buffer),
  }
  }
 }
@@ -257,6 +267,7 @@ fn value_as_f64(value: Value) -> f64 {
 }
 
 #[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
 struct ForeignFunction {
  parameters: Vec<NativeType>,
  result: NativeType,
@@ -293,20 +304,32 @@ where
  Ok(state.resource_table.add(resource))
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct FfiCallArgs {
  rid: ResourceId,
  symbol: String,
  parameters: Vec<Value>,
+ buffers: Vec<ZeroCopyBuf>,
 }
 
 fn ffi_call(args: FfiCallArgs, symbol: &Symbol) -> Result<Value, AnyError> {
+ let buffers: Vec<&[u8]> =
+ args.buffers.iter().map(|buffer| &buffer[..]).collect();
+
  let native_values = symbol
  .parameter_types
  .iter()
  .zip(args.parameters.into_iter())
- .map(|(&native_type, value)| NativeValue::new(native_type, value))
+ .map(|(&native_type, value)| {
+ if let NativeType::Buffer = native_type {
+ let idx: usize = value_as_uint(value);
+ let ptr = buffers[idx].as_ptr();
+ NativeValue::buffer(ptr)
+ } else {
+ NativeValue::new(native_type, value)
+ }
+ })
  .collect::<Vec<_>>();
 
  let call_args = symbol
@@ -358,6 +381,7 @@ fn ffi_call(args: FfiCallArgs, symbol: &Symbol) -> Result<Value, AnyError> {
  NativeType::F64 => {
  json!(unsafe { symbol.cif.call::<f64>(symbol.ptr, &call_args) })
  }
+ NativeType::Buffer => unreachable!(),
  })
 }
 

test_ffi/src/lib.rs

@@ -1,3 +1,5 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
 use std::thread::sleep;
 use std::time::Duration;
 
@@ -6,6 +8,13 @@ pub extern "C" fn print_something() {
  println!("something");
 }
 
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn print_buffer(ptr: *const u8, len: usize) {
+ let buf = unsafe { std::slice::from_raw_parts(ptr, len) };
+ println!("{:?}", buf);
+}
+
 #[no_mangle]
 pub extern "C" fn add_u32(a: u32, b: u32) -> u32 {
  a + b
@@ -51,3 +60,10 @@ pub extern "C" fn sleep_blocking(ms: u64) {
  let duration = Duration::from_millis(ms);
  sleep(duration);
 }
+
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn nonblocking_buffer(ptr: *const u8, len: usize) {
+ let buf = unsafe { std::slice::from_raw_parts(ptr, len) };
+ assert_eq!(buf, vec![1, 2, 3, 4, 5, 6, 7, 8]);
+}

test_ffi/tests/integration_tests.rs

@@ -38,6 +38,7 @@ fn basic() {
  assert!(output.status.success());
  let expected = "\
  something\n\
+ [1, 2, 3, 4, 5, 6, 7, 8]\n\
  579\n\
  579\n\
  579\n\

test_ffi/tests/test.js

@@ -12,6 +12,7 @@ const libPath = `${targetDir}/${libPrefix}test_ffi.${libSuffix}`;
 const resourcesPre = Deno.resources();
 const dylib = Deno.dlopen(libPath, {
  "print_something": { parameters: [], result: "void" },
+ "print_buffer": { parameters: ["buffer", "usize"], result: "void" },
  "add_u32": { parameters: ["u32", "u32"], result: "u32" },
  "add_i32": { parameters: ["i32", "i32"], result: "i32" },
  "add_u64": { parameters: ["u64", "u64"], result: "u64" },
@@ -21,9 +22,16 @@ const dylib = Deno.dlopen(libPath, {
  "add_f32": { parameters: ["f32", "f32"], result: "f32" },
  "add_f64": { parameters: ["f64", "f64"], result: "f64" },
  "sleep_blocking": { parameters: ["u64"], result: "void", nonblocking: true },
+ "nonblocking_buffer": {
+ parameters: ["buffer", "usize"],
+ result: "void",
+ nonblocking: true,
+ },
 });
 
 dylib.symbols.print_something();
+const buffer = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+dylib.symbols.print_buffer(buffer, buffer.length);
 console.log(dylib.symbols.add_u32(123, 456));
 console.log(dylib.symbols.add_i32(123, 456));
 console.log(dylib.symbols.add_u64(123, 456));
@@ -34,6 +42,30 @@ console.log(dylib.symbols.add_f32(123.123, 456.789));
 console.log(dylib.symbols.add_f64(123.123, 456.789));
 
 // Test non blocking calls
+
+function deferred() {
+ let methods;
+ const promise = new Promise((resolve, reject) => {
+ methods = {
+ async resolve(value) {
+ await value;
+ resolve(value);
+ },
+ reject(reason) {
+ reject(reason);
+ },
+ };
+ });
+ return Object.assign(promise, methods);
+}
+
+const promise = deferred();
+const buffer2 = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+dylib.symbols.nonblocking_buffer(buffer2, buffer2.length).then(() => {
+ promise.resolve();
+});
+await promise;
+
 const start = performance.now();
 dylib.symbols.sleep_blocking(100).then(() => {
  console.log("After");