From 0b440db7729e55f64f141554133e479cc189d2d6 Mon Sep 17 00:00:00 2001 From: snek Date: Wed, 26 Jun 2024 19:49:06 -0700 Subject: [PATCH] feat: rust bindgen and publish flow (#1507) --- .github/workflows/ci.yml | 68 +++++++++++++------- Cargo.lock | 111 ++++++++++++++++++++++++++++++++- Cargo.toml | 2 + build.rs | 128 ++++++++++++++++++++++++++++---------- src/binding.cc | 3 - src/binding.hpp | 12 ++++ src/binding.rs | 4 ++ src/lib.rs | 1 + src/script.rs | 5 +- tools/get_bindgen_args.py | 21 +++++++ 10 files changed, 295 insertions(+), 60 deletions(-) create mode 100644 src/binding.hpp create mode 100644 src/binding.rs create mode 100644 tools/get_bindgen_args.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 472a856a14..3106aec10f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,7 +202,7 @@ jobs: if: startsWith(matrix.config.os, 'ubuntu') run: | sudo apt-get install -y clang-format - clang-format --Werror --dry-run src/binding.cc + clang-format --verbose --Werror --dry-run src/*.cc src/*.hpp - name: Test (ASAN) env: @@ -231,11 +231,13 @@ jobs: - name: Prepare binary publish if: matrix.config.variant == 'debug' || matrix.config.variant == 'release' - run: gzip -9c - target/${{ matrix.config.target }}/${{ matrix.config.variant }}/gn_out/obj/${{ env.LIB_NAME }}.${{ env.LIB_EXT }} > - target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz && + run: | + gzip -9c target/${{ matrix.config.target }}/${{ matrix.config.variant }}/gn_out/obj/${{ env.LIB_NAME }}.${{ env.LIB_EXT }} > target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz ls -l target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz + cp target/${{ matrix.config.target }}/${{ matrix.config.variant}}/gn_out/src_binding.rs target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs + ls -l target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs + - name: Binary publish uses: softprops/action-gh-release@v0.1.15 if: >- @@ -247,25 +249,49 @@ jobs: with: files: target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz - # TODO: add clang-format and maybe cpplint. + - name: Upload CI artifacts + uses: actions/upload-artifact@v4 + with: + name: src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs + path: target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs - - name: Cargo Package - if: >- - github.repository == 'denoland/rusty_v8' && - startsWith(matrix.config.target , 'x86_64') && - matrix.config.variant == 'debug' && - runner.os == 'Linux' - run: cargo package -vv --locked + publish: + needs: build + runs-on: ${{ github.repository == 'denoland/rusty_v8' && 'ubuntu-22.04-xl' || 'ubuntu-22.04' }} + steps: + - name: Configure git + run: git config --global core.symlinks true + + - name: Clone repository + uses: actions/checkout@v4 + with: + fetch-depth: 10 + submodules: recursive + + - name: Install rust + uses: dsherret/rust-toolchain-file@v1 + + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + architecture: x64 + + - name: Download CI artifacts + uses: actions/download-artifact@v4 + with: + path: gen + pattern: src_binding_*.rs + merge-multiple: true - name: Publish - # Only publish on x64 linux when there's a git tag: - if: >- - startsWith(github.ref, 'refs/tags/') && - github.repository == 'denoland/rusty_v8' && - startsWith(matrix.config.target , 'x86_64') && - !endsWith(matrix.config.target , 'android') && - matrix.config.variant == 'debug' && - runner.os == 'Linux' + if: github.repository == 'denoland/rusty_v8' env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: cargo publish -vv + DRY_RUN: ${{ startsWith(github.ref, 'refs/tags/') == false }} + run: | + args="-vv --locked --allow-dirty" + if [ "$DRY_RUN" = "true" ]; then + args="$args --dry-run" + fi + cargo publish $args diff --git a/Cargo.lock b/Cargo.lock index 89032644cb..4ff6f96ab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.60", + "which 4.4.2", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -153,6 +176,15 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -171,6 +203,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.3", +] + [[package]] name = "cocoa" version = "0.24.1" @@ -604,6 +647,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -641,6 +693,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.153" @@ -736,6 +794,12 @@ dependencies = [ "objc", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.3" @@ -866,6 +930,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -978,6 +1052,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.60", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -1081,6 +1165,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1382d1f0a252c4bf97dc20d979a2fdd05b024acd7c2ed0f7595d7817666a157" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.33" @@ -1167,6 +1257,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "slotmap" version = "1.0.7" @@ -1353,6 +1449,7 @@ name = "v8" version = "0.94.0" dependencies = [ "align-data", + "bindgen", "bitflags 2.5.0", "bytes", "fslock", @@ -1362,7 +1459,7 @@ dependencies = [ "once_cell", "rustversion", "trybuild", - "which", + "which 6.0.1", ] [[package]] @@ -1615,6 +1712,18 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "which" version = "6.0.1" diff --git a/Cargo.toml b/Cargo.toml index 9835e31bf6..16c79455df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ gzip-header = "1.0.0" fslock = "0.2" which = "6" home = "0" +bindgen = "0.69" [dev-dependencies] miniz_oxide = "0.7.3" @@ -107,6 +108,7 @@ trybuild = "1.0.96" which = "6" home = "0" rustversion = "1" +bindgen = "0.69" [[example]] name = "hello_world" diff --git a/build.rs b/build.rs index 48221b6e54..af1d02e2a3 100644 --- a/build.rs +++ b/build.rs @@ -72,6 +72,7 @@ fn main() { // Early exit if is_cargo_doc || is_rls { + print_prebuilt_src_binding_path(); return; } @@ -79,6 +80,10 @@ fn main() { // Don't attempt rebuild but link if is_trybuild { + println!( + "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}", + env::var("RUSTY_V8_SRC_BINDING_PATH").unwrap() + ); return; } @@ -92,14 +97,22 @@ fn main() { }; // Build from source - if env::var_os("V8_FROM_SOURCE").is_some() { + if env_bool("V8_FROM_SOURCE") { if is_asan && std::env::var_os("OPT_LEVEL").unwrap_or_default() == "0" { panic!("v8 crate cannot be compiled with OPT_LEVEL=0 and ASAN.\nTry `[profile.dev.package.v8] opt-level = 1`.\nAborting before miscompilations cause issues."); } - return build_v8(is_asan); + // cargo publish doesn't like pyc files. + env::set_var("PYTHONDONTWRITEBYTECODE", "1"); + + build_v8(is_asan); + build_binding(); + + return; } + print_prebuilt_src_binding_path(); + // utilize a lockfile to prevent linking of // only partially downloaded static library. let root = env::current_dir().unwrap(); @@ -119,12 +132,38 @@ fn main() { lockfile.unlock().expect("Couldn't unlock lockfile"); } +fn build_binding() { + let output = Command::new(python()) + .arg("./tools/get_bindgen_args.py") + .arg("--gn-out") + .arg(build_dir().join("gn_out")) + .output() + .unwrap(); + let args = String::from_utf8(output.stdout).unwrap(); + let args = args.split('\0').collect::>(); + + let bindings = bindgen::Builder::default() + .header("src/binding.hpp") + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .clang_args(["-x", "c++", "-std=c++20", "-Iv8/include"]) + .clang_args(args) + .allowlist_item("RUST_.*") + .generate() + .expect("Unable to generate bindings"); + + let out_path = build_dir().join("gn_out").join("src_binding.rs"); + println!( + "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}", + out_path.display() + ); + bindings + .write_to_file(out_path) + .expect("Couldn't write bindings!"); +} + fn build_v8(is_asan: bool) { env::set_var("DEPOT_TOOLS_WIN_TOOLCHAIN", "0"); - // cargo publish doesn't like pyc files. - env::set_var("PYTHONDONTWRITEBYTECODE", "1"); - // git submodule update --init --recursive let libcxx_src = PathBuf::from("buildtools/third_party/libc++/trunk/src"); if !libcxx_src.is_dir() { @@ -369,49 +408,48 @@ fn download_ninja_gn_binaries() { env::set_var("NINJA", ninja); } -fn static_lib_url() -> String { - if let Ok(custom_archive) = env::var("RUSTY_V8_ARCHIVE") { - return custom_archive; - } - let default_base = "https://github.com/denoland/rusty_v8/releases/download"; - let base = - env::var("RUSTY_V8_MIRROR").unwrap_or_else(|_| default_base.into()); - let version = env::var("CARGO_PKG_VERSION").unwrap(); - let target = env::var("TARGET").unwrap(); +fn prebuilt_profile() -> &'static str { let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); // Note: we always use the release build on windows. if target_os == "windows" { - return format!("{}/v{}/rusty_v8_release_{}.lib.gz", base, version, target); + return "release"; } // Use v8 in release mode unless $V8_FORCE_DEBUG=true - let profile = match env_bool("V8_FORCE_DEBUG") { + match env_bool("V8_FORCE_DEBUG") { true => "debug", _ => "release", - }; - format!( - "{}/v{}/librusty_v8_{}_{}.a.gz", - base, version, profile, target - ) -} - -fn env_bool(key: &str) -> bool { - matches!( - env::var(key).unwrap_or_default().as_str(), - "true" | "1" | "yes" - ) + } } -fn static_lib_name() -> &'static str { +fn static_lib_name(suffix: &str) -> String { let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); if target_os == "windows" { - "rusty_v8.lib" + format!("rusty_v8{suffix}.lib") } else { - "librusty_v8.a" + format!("librusty_v8{suffix}.a") + } +} + +fn static_lib_url() -> String { + if let Ok(custom_archive) = env::var("RUSTY_V8_ARCHIVE") { + return custom_archive; } + let default_base = "https://github.com/denoland/rusty_v8/releases/download"; + let base = + env::var("RUSTY_V8_MIRROR").unwrap_or_else(|_| default_base.into()); + let version = env::var("CARGO_PKG_VERSION").unwrap(); + let target = env::var("TARGET").unwrap(); + let profile = prebuilt_profile(); + format!( + "{}/v{}/{}.gz", + base, + version, + static_lib_name(&format!("_{}_{}", profile, target)), + ) } fn static_lib_path() -> PathBuf { - static_lib_dir().join(static_lib_name()) + static_lib_dir().join(static_lib_name("")) } fn static_checksum_path() -> PathBuf { @@ -666,6 +704,19 @@ fn print_link_flags() { } } +fn print_prebuilt_src_binding_path() { + let target = env::var("TARGET").unwrap(); + let profile = prebuilt_profile(); + let src_binding_path = get_dirs(None) + .root + .join("gen") + .join(format!("src_binding_{}_{}.rs", profile, target)); + println!( + "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}", + src_binding_path.display() + ); +} + // Chromium depot_tools contains helpers // which delegate to the "relevant" `buildtools` // directory when invoked, so they don't count. @@ -875,6 +926,7 @@ pub fn maybe_gen(manifest_dir: &str, gn_args: GnArgs) -> PathBuf { .arg(format!("--script-executable={}", python())) .arg("gen") .arg(&gn_out_dir) + .arg("--ide=json") .arg("--args=".to_owned() + &args) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -914,8 +966,9 @@ fn rerun_if_changed(out_dir: &Path, maybe_env: Option, target: &str) { let deps = ninja_get_deps(out_dir, maybe_env, target); for d in deps { let p = out_dir.join(d); - assert!(p.exists(), "Path doesn't exist: {:?}", p); - println!("cargo:rerun-if-changed={}", p.display()); + if p.exists() { + println!("cargo:rerun-if-changed={}", p.display()); + } } } @@ -975,6 +1028,13 @@ pub fn parse_ninja_graph(s: &str) -> HashSet { out } +fn env_bool(key: &str) -> bool { + matches!( + env::var(key).unwrap_or_default().as_str(), + "true" | "1" | "yes" + ) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/binding.cc b/src/binding.cc index 4abc4cbe3c..82d125294f 100644 --- a/src/binding.cc +++ b/src/binding.cc @@ -44,9 +44,6 @@ static_assert(sizeof(two_pointers_t) == sizeof(std::shared_ptr), "std::shared_ptr size mismatch"); -static_assert(sizeof(v8::ScriptOrigin) <= sizeof(size_t) * 8, - "ScriptOrigin size mismatch"); - static_assert(sizeof(v8::HandleScope) == sizeof(size_t) * 3, "HandleScope size mismatch"); diff --git a/src/binding.hpp b/src/binding.hpp new file mode 100644 index 0000000000..5fee9723cd --- /dev/null +++ b/src/binding.hpp @@ -0,0 +1,12 @@ +#include + +/** + * Types defined here will be compiled with bindgen + * and made available in `crate::binding` in rust. + */ + +// TODO: In the immediate term, cppgc definitions will go here. +// In the future we should migrate over the rest of our SIZE definitions, +// and eventually entire structs and functions. + +static size_t RUST_v8__ScriptOrigin_SIZE = sizeof(v8::ScriptOrigin); diff --git a/src/binding.rs b/src/binding.rs new file mode 100644 index 0000000000..3ef84eac19 --- /dev/null +++ b/src/binding.rs @@ -0,0 +1,4 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +include!(env!("RUSTY_V8_SRC_BINDING_PATH")); diff --git a/src/lib.rs b/src/lib.rs index 370eeeb50c..feb40d2ec2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ extern crate bitflags; mod array_buffer; mod array_buffer_view; mod bigint; +mod binding; mod context; pub mod cppgc; mod data; diff --git a/src/script.rs b/src/script.rs index 310f4e0e97..04703152f3 100644 --- a/src/script.rs +++ b/src/script.rs @@ -13,7 +13,10 @@ use crate::Value; /// The origin, within a file, of a script. #[repr(C)] #[derive(Debug)] -pub struct ScriptOrigin<'s>([usize; 8], PhantomData<&'s ()>); +pub struct ScriptOrigin<'s>( + [u8; crate::binding::RUST_v8__ScriptOrigin_SIZE], + PhantomData<&'s ()>, +); extern "C" { fn v8__Script__Compile( diff --git a/tools/get_bindgen_args.py b/tools/get_bindgen_args.py new file mode 100644 index 0000000000..0d080956c7 --- /dev/null +++ b/tools/get_bindgen_args.py @@ -0,0 +1,21 @@ +import argparse +import json +import os + +parser = argparse.ArgumentParser(description='Generate args for bindgen') +parser.add_argument('--gn-out', help='GN out directory') +args = parser.parse_args() + +with open(os.path.join(args.gn_out, 'project.json')) as project_json: + project = json.load(project_json) + +target = project['targets']['//v8:v8_headers'] + +assert '//v8:cppgc_headers' in target['deps'] + +args = [] + +for define in target['defines']: + args.append(f'-D{define}') + +print('\0'.join(args), end="")