Introduce #[napi] procedural macro to automation development boiler…

…plate (#696)

* napi procedural macro for basic rust/JavaScript types
* introduce the `compat-mode` for `napi` and `napi-derive` crates for backward compatible
* remove #[inline] and let compiler to decide the inline behavior
* cli now can produce the `.d.ts` file for native binding
* many tests and example for the new procedural macro

Co-authored-by: LongYinan <[email protected]>

.eslintrc.yml

@@ -196,11 +196,12 @@ overrides:
  parserOptions:
  project: ./tsconfig.json
  - files:
- - ./test_module/**/*.{ts,js}
+ - ./examples/**/*.{ts,js}
  plugins:
  - '@typescript-eslint'
  parserOptions:
- project: ./test_module/tsconfig.json
+ project:
+ - ./examples/tsconfig.json
 
  - files:
  - ./bench/**/*.{ts,js}

.github/workflows/linux-aarch64-musl.yaml

@@ -77,7 +77,7 @@ jobs:
 
  - name: Cross build native tests
  run: |
- docker run -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry -v $(pwd):/napi-rs -w /napi-rs -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc builder yarn --cwd ./test_module build --target aarch64-unknown-linux-musl
+ docker run -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry -v $(pwd):/napi-rs -w /napi-rs -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc builder sh -c "yarn --cwd ./examples/napi-compat-mode build --target aarch64-unknown-linux-musl && yarn --cwd ./examples/napi build --target aarch64-unknown-linux-musl"
 
  - name: Setup and run tests
  uses: docker:https://multiarch/alpine:aarch64-latest-stable

.github/workflows/napi3.yaml

@@ -70,7 +70,8 @@ jobs:
 
  - name: Unit tests
  run: |
- yarn --cwd ./test_module --ignore-engines build-napi3
+ yarn --cwd ./examples/napi-compat-mode --ignore-engines build-napi3
+ yarn --cwd ./examples/napi --ignore-engines build-napi3
  yarn --ignore-engines test
  env:
  RUST_BACKTRACE: 1

.github/workflows/windows-i686.yml

@@ -40,7 +40,7 @@ jobs:
  - name: Install
  uses: actions-rs/toolchain@v1
  with:
- toolchain: 1.51.0
+ toolchain: stable
  profile: minimal
  override: true
 
@@ -72,7 +72,8 @@ jobs:
 
  - name: Build Tests
  run: |
- yarn --cwd ./test_module build-i686
+ yarn --cwd ./examples/napi-compat-mode build-i686
+ yarn --cwd ./examples/napi build-i686
  yarn test
  env:
  RUST_BACKTRACE: 1

Cargo.toml

@@ -1,11 +1,12 @@
 [workspace]
 members = [
- "./build",
- "./napi",
- "./napi-derive",
- "./napi-derive-example",
- "./sys",
- "./test_module",
+ "./crates/backend",
+ "./crates/build",
+ "./crates/macro",
+ "./crates/napi",
+ "./crates/sys",
+ "./examples/napi",
+ "./examples/napi-compat-mode",
  "./bench",
  "./memory-testing",
 ]

ava.config.js

@@ -1,9 +1,9 @@
 const configuration = {
  extensions: ['ts', 'tsx'],
- files: ['test_module/__test__/**/*.spec.ts', 'cli/__test__/**/*.spec.ts'],
+ files: ['examples/**/__test__/**/*.spec.ts'],
  require: ['ts-node/register/transpile-only'],
  environmentVariables: {
- TS_NODE_PROJECT: './test_module/tsconfig.json',
+ TS_NODE_PROJECT: './examples/tsconfig.json',
  },
  timeout: '1m',
 }

bench/Cargo.toml

@@ -8,13 +8,13 @@ version = "0.1.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-napi = {path = "../napi", features = ["tokio_rt", "serde-json"]}
-napi-derive = {path = "../napi-derive"}
+napi = {path = "../crates/napi", features = ["tokio_rt", "serde-json", "compat-mode"]}
+napi-derive = {path = "../crates/macro", features = ["compat-mode"]}
 serde = "1"
 serde_json = "1"
 
 [target.'cfg(all(target_arch = "x86_64", not(target_env = "musl")))'.dependencies]
 mimalloc = {version = "0.1"}
 
 [build-dependencies]
-napi-build = {path = "../build"}
+napi-build = {path = "../crates/build"}

bench/src/get_set_property.rs

@@ -44,16 +44,16 @@ pub fn register_js(exports: &mut JsObject, env: &Env) -> Result<()> {
  "TestClass",
  test_class_constructor,
  &[
- Property::new(env, "miterNative")?
+ Property::new("miterNative")?
  .with_getter(get_miter_native)
  .with_setter(set_miter_native),
- Property::new(env, "miter")?
+ Property::new("miter")?
  .with_getter(get_miter)
  .with_setter(set_miter),
- Property::new(env, "lineJoinNative")?
+ Property::new("lineJoinNative")?
  .with_getter(get_line_join_native)
  .with_setter(set_line_join_native),
- Property::new(env, "lineJoin")?
+ Property::new("lineJoin")?
  .with_getter(get_line_join)
  .with_setter(set_line_join),
  ],

cli/src/build.ts

@@ -14,6 +14,7 @@ import {
  mkdirAsync,
  readFileAsync,
  unlinkAsync,
+ writeFileAsync,
 } from './utils'
 
 const debug = debugFactory('build')
@@ -50,7 +51,7 @@ export class BuildCommand extends Command {
  ? join(process.cwd(), this.cargoCwd)
  : process.cwd()
  const releaseFlag = this.isRelease ? `--release` : ''
- const targetFLag = this.targetTripleDir
+ const targetFlag = this.targetTripleDir
  ? `--target ${this.targetTripleDir}`
  : ''
  const featuresFlag = this.features ? `--features ${this.features}` : ''
@@ -64,16 +65,20 @@ export class BuildCommand extends Command {
  debug(`Current triple is: ${chalk.green(triple.raw)}`)
  const externalFlags = [
  releaseFlag,
- targetFLag,
+ targetFlag,
  featuresFlag,
  this.cargoFlags,
  ]
  .filter((flag) => Boolean(flag))
  .join(' ')
  const cargoCommand = `cargo build ${externalFlags}`
+ const intermediateTypeFile = join(__dirname, `type_def.${Date.now()}.tmp`)
  debug(`Run ${chalk.green(cargoCommand)}`)
  execSync(cargoCommand, {
- env: process.env,
+ env: {
+ ...process.env,
+ TYPE_DEF_TMP_PATH: intermediateTypeFile,
+ },
  stdio: 'inherit',
  cwd,
  })
@@ -190,6 +195,11 @@ export class BuildCommand extends Command {
 
  debug(`Write binary content to [${chalk.yellowBright(distModulePath)}]`)
  await copyFileAsync(sourcePath, distModulePath)
+
+ await processIntermediateTypeFile(
+ intermediateTypeFile,
+ join(this.destDir ?? '.', 'type.d.ts'),
+ )
  }
 }
 
@@ -205,3 +215,60 @@ async function findUp(dir = process.cwd()): Promise<string | null> {
  dirs.pop()
  return findUp(dirs.join(sep))
 }
+
+interface TypeDef {
+ kind: 'fn' | 'struct' | 'impl' | 'enum'
+ name: string
+ def: string
+}
+
+async function processIntermediateTypeFile(source: string, target: string) {
+ if (!(await existsAsync(source))) {
+ debug(`do not find tmp type file. skip type generation`)
+ return
+ }
+
+ const tmpFile = await readFileAsync(source, 'utf8')
+ const lines = tmpFile
+ .split('\n')
+ .map((line) => line.trim())
+ .filter(Boolean)
+ let dts = ''
+ const classes = new Map<string, string>()
+ const impls = new Map<string, string>()
+
+ lines.forEach((line) => {
+ const def = JSON.parse(line) as TypeDef
+
+ switch (def.kind) {
+ case 'fn':
+ case 'enum':
+ dts += def.def + '\n'
+ break
+ case 'struct':
+ classes.set(def.name, def.def)
+ break
+ case 'impl':
+ impls.set(def.name, def.def)
+ }
+ })
+
+ for (const [name, def] of impls.entries()) {
+ const classDef = classes.get(name)
+
+ dts += `export class ${name} {
+ ${(classDef ?? '')
+ .split('\n')
+ .map((line) => line.trim())
+ .join('\n ')}
+ ${def
+ .split('\n')
+ .map((line) => line.trim())
+ .join('\n ')}
+}
+`
+ }
+
+ await unlinkAsync(source)
+ await writeFileAsync(target, dts, 'utf8')
+}

crates/backend/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "napi-derive-backend"
+version = "0.1.0"
+edition = "2018"
+
+[features]
+type-def = ["regex", "once_cell"]
+strict = []
+
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = {version = "1.0", features = ["fold", "full"]}
+convert_case = "0.4.0"
+
+[dependencies.regex]
+optional = true
+version = "1"
+
+[dependencies.once_cell]
+optional = true
+version = "1"

crates/backend/README.md

@@ -0,0 +1,3 @@
+# napi-derive-backend
+
+Take care the ast parsing from `napi-derive` and generate "bridge" runtime code for both nodejs and rust.

crates/backend/src/ast.rs

@@ -0,0 +1,85 @@
+use proc_macro2::Ident;
+use syn::Attribute;
+
+#[derive(Debug, Clone)]
+pub struct NapiFn {
+ pub name: Ident,
+ pub js_name: String,
+ pub attrs: Vec<Attribute>,
+ pub args: Vec<NapiFnArgKind>,
+ pub ret: Option<syn::Type>,
+ pub is_async: bool,
+ pub fn_self: Option<FnSelf>,
+ pub kind: FnKind,
+ pub vis: syn::Visibility,
+ pub parent: Option<Ident>,
+ pub strict: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct CallbackArg {
+ pub pat: Box<syn::Pat>,
+ pub args: Vec<syn::Type>,
+ pub ret: Option<syn::Type>,
+}
+
+#[derive(Debug, Clone)]
+pub enum NapiFnArgKind {
+ PatType(Box<syn::PatType>),
+ Callback(Box<CallbackArg>),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum FnKind {
+ Normal,
+ Constructor,
+ Getter,
+ Setter,
+}
+
+#[derive(Debug, Clone)]
+pub enum FnSelf {
+ Value,
+ Ref,
+ MutRef,
+}
+
+#[derive(Debug, Clone)]
+pub struct NapiStruct {
+ pub name: Ident,
+ pub js_name: String,
+ pub vis: syn::Visibility,
+ pub fields: Vec<NapiStructField>,
+ pub is_tuple: bool,
+ pub gen_default_ctor: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct NapiStructField {
+ pub name: syn::Member,
+ pub js_name: String,
+ pub ty: syn::Type,
+ pub getter: bool,
+ pub setter: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct NapiImpl {
+ pub name: Ident,
+ pub js_name: String,
+ pub items: Vec<NapiFn>,
+}
+
+#[derive(Debug, Clone)]
+pub struct NapiEnum {
+ pub name: Ident,
+ pub js_name: String,
+ pub variants: Vec<NapiEnumVariant>,
+}
+
+#[derive(Debug, Clone)]
+pub struct NapiEnumVariant {
+ pub name: Ident,
+ pub val: i32,
+ pub comments: Vec<String>,
+}

crates/backend/src/codegen.rs

@@ -0,0 +1,28 @@
+use proc_macro2::{Ident, Span, TokenStream};
+
+use crate::BindgenResult;
+
+mod r#enum;
+mod r#fn;
+mod r#struct;
+
+pub trait TryToTokens {
+ fn try_to_tokens(&self, tokens: &mut TokenStream) -> BindgenResult<()>;
+
+ fn try_to_token_stream(&self) -> BindgenResult<TokenStream> {
+ let mut tokens = TokenStream::default();
+ self.try_to_tokens(&mut tokens)?;
+
+ Ok(tokens)
+ }
+}
+
+fn get_intermediate_ident(name: &str) -> Ident {
+ let new_name = format!("__napi__{}", name);
+ Ident::new(&new_name, Span::call_site())
+}
+
+fn get_register_ident(name: &str) -> Ident {
+ let new_name = format!("__napi_register__{}", name);
+ Ident::new(&new_name, Span::call_site())
+}