Loader: support .wasm imports (denoland#3328)

* loader: support .wasm imports

* http_server: true

* Support named exports

* Clippy

.eslintignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 std/deno.d.ts
 std/prettier/vendor

.prettierignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 cli/tests/badly_formatted.js
 cli/tests/top_level_for_await.js

cli/Cargo.toml

@@ -27,6 +27,7 @@ deno_typescript = { path = "../deno_typescript", version = "0.23.0" }
 
 ansi_term = "0.12.1"
 atty = "0.2.13"
+base64 = "0.11.0"
 byteorder = "1.3.2"
 clap = "2.33.0"
 dirs = "2.0.2"

cli/compilers/mod.rs

@@ -5,10 +5,12 @@ use futures::Future;
 mod js;
 mod json;
 mod ts;
+mod wasm;
 
 pub use js::JsCompiler;
 pub use json::JsonCompiler;
 pub use ts::TsCompiler;
+pub use wasm::WasmCompiler;
 
 #[derive(Debug, Clone)]
 pub struct CompiledModule {

cli/compilers/wasm.rs

@@ -0,0 +1,174 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use crate::compilers::CompiledModule;
+use crate::compilers::CompiledModuleFuture;
+use crate::file_fetcher::SourceFile;
+use crate::global_state::ThreadSafeGlobalState;
+use crate::startup_data;
+use crate::state::*;
+use crate::worker::Worker;
+use deno::Buf;
+use futures::Future;
+use futures::IntoFuture;
+use serde_derive::Deserialize;
+use serde_json;
+use std::collections::HashMap;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex};
+use url::Url;
+
+// TODO(kevinkassimo): This is a hack to encode/decode data as base64 string.
+// (Since Deno namespace might not be available, Deno.read can fail).
+// Binary data is already available through source_file.source_code.
+// If this is proven too wasteful in practice, refactor this.
+
+// Ref: https://webassembly.github.io/esm-integration/js-api/index.html#esm-integration
+// https://github.com/nodejs/node/blob/35ec01097b2a397ad0a22aac536fe07514876e21/lib/internal/modules/esm/translators.js#L190-L210
+
+// Dynamically construct JS wrapper with custom static imports and named exports.
+// Boots up an internal worker to resolve imports/exports through query from V8.
+
+static WASM_WRAP: &str = include_str!("./wasm_wrap.js");
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct WasmModuleInfo {
+ import_list: Vec<String>,
+ export_list: Vec<String>,
+}
+
+#[derive(Default)]
+pub struct WasmCompiler {
+ cache: Arc<Mutex<HashMap<Url, CompiledModule>>>,
+}
+
+impl WasmCompiler {
+ /// Create a new V8 worker with snapshot of WASM compiler and setup compiler's runtime.
+ fn setup_worker(global_state: ThreadSafeGlobalState) -> Worker {
+ let (int, ext) = ThreadSafeState::create_channels();
+ let worker_state =
+ ThreadSafeState::new(global_state.clone(), None, true, int)
+ .expect("Unable to create worker state");
+
+ // Count how many times we start the compiler worker.
+ global_state
+ .metrics
+ .compiler_starts
+ .fetch_add(1, Ordering::SeqCst);
+
+ let mut worker = Worker::new(
+ "WASM".to_string(),
+ startup_data::compiler_isolate_init(),
+ worker_state,
+ ext,
+ );
+ worker.execute("denoMain('WASM')").unwrap();
+ worker.execute("workerMain()").unwrap();
+ worker.execute("wasmCompilerMain()").unwrap();
+ worker
+ }
+
+ pub fn compile_async(
+ self: &Self,
+ global_state: ThreadSafeGlobalState,
+ source_file: &SourceFile,
+ ) -> Box<CompiledModuleFuture> {
+ let cache = self.cache.clone();
+ let maybe_cached = { cache.lock().unwrap().get(&source_file.url).cloned() };
+ if let Some(m) = maybe_cached {
+ return Box::new(futures::future::ok(m.clone()));
+ }
+ let cache_ = self.cache.clone();
+
+ debug!(">>>>> wasm_compile_async START");
+ let base64_data = base64::encode(&source_file.source_code);
+ let worker = WasmCompiler::setup_worker(global_state.clone());
+ let worker_ = worker.clone();
+ let url = source_file.url.clone();
+
+ let fut = worker
+ .post_message(
+ serde_json::to_string(&base64_data)
+ .unwrap()
+ .into_boxed_str()
+ .into_boxed_bytes(),
+ )
+ .into_future()
+ .then(move |_| worker)
+ .then(move |result| {
+ if let Err(err) = result {
+ // TODO(ry) Need to forward the error instead of exiting.
+ eprintln!("{}", err.to_string());
+ std::process::exit(1);
+ }
+ debug!("Sent message to worker");
+ worker_.get_message()
+ })
+ .map_err(|_| panic!("not handled"))
+ .and_then(move |maybe_msg: Option<Buf>| {
+ debug!("Received message from worker");
+ let json_msg = maybe_msg.unwrap();
+ let module_info: WasmModuleInfo =
+ serde_json::from_slice(&json_msg).unwrap();
+ debug!("WASM module info: {:#?}", &module_info);
+ let code = wrap_wasm_code(
+ &base64_data,
+ &module_info.import_list,
+ &module_info.export_list,
+ );
+ debug!("Generated code: {}", &code);
+ let module = CompiledModule {
+ code,
+ name: url.to_string(),
+ };
+ {
+ cache_.lock().unwrap().insert(url.clone(), module.clone());
+ }
+ debug!("<<<<< wasm_compile_async END");
+ Ok(module)
+ });
+ Box::new(fut)
+ }
+}
+
+fn build_single_import(index: usize, origin: &str) -> String {
+ let origin_json = serde_json::to_string(origin).unwrap();
+ format!(
+ r#"import * as m{} from {};
+importObject[{}] = m{};
+"#,
+ index, &origin_json, &origin_json, index
+ )
+}
+
+fn build_imports(imports: &[String]) -> String {
+ let mut code = String::from("");
+ for (index, origin) in imports.iter().enumerate() {
+ code.push_str(&build_single_import(index, origin));
+ }
+ code
+}
+
+fn build_single_export(name: &str) -> String {
+ format!("export const {} = instance.exports.{};\n", name, name)
+}
+
+fn build_exports(exports: &[String]) -> String {
+ let mut code = String::from("");
+ for e in exports {
+ code.push_str(&build_single_export(e));
+ }
+ code
+}
+
+fn wrap_wasm_code(
+ base64_data: &str,
+ imports: &[String],
+ exports: &[String],
+) -> String {
+ let imports_code = build_imports(imports);
+ let exports_code = build_exports(exports);
+ String::from(WASM_WRAP)
+ .replace("//IMPORTS\n", &imports_code)
+ .replace("//EXPORTS\n", &exports_code)
+ .replace("BASE64_DATA", base64_data)
+}

cli/compilers/wasm_wrap.js

@@ -0,0 +1,19 @@
+const importObject = Object.create(null);
+//IMPORTS
+
+function base64ToUint8Array(data) {
+ const binString = window.atob(data);
+ const size = binString.length;
+ const bytes = new Uint8Array(size);
+ for (let i = 0; i < size; i++) {
+ bytes[i] = binString.charCodeAt(i);
+ }
+ return bytes;
+}
+
+const buffer = base64ToUint8Array("BASE64_DATA");
+const compiled = await WebAssembly.compile(buffer);
+
+const instance = new WebAssembly.Instance(compiled, importObject);
+
+//EXPORTS

cli/file_fetcher.rs

@@ -491,6 +491,7 @@ fn map_file_extension(path: &Path) -> msg::MediaType {
  Some("jsx") => msg::MediaType::JSX,
  Some("mjs") => msg::MediaType::JavaScript,
  Some("json") => msg::MediaType::Json,
+ Some("wasm") => msg::MediaType::Wasm,
  _ => msg::MediaType::Unknown,
  },
  }
@@ -1503,6 +1504,10 @@ mod tests {
  map_file_extension(Path::new("foo/bar.json")),
  msg::MediaType::Json
  );
+ assert_eq!(
+ map_file_extension(Path::new("foo/bar.wasm")),
+ msg::MediaType::Wasm
+ );
  assert_eq!(
  map_file_extension(Path::new("foo/bar.txt")),
  msg::MediaType::Unknown
@@ -1544,6 +1549,10 @@ mod tests {
  map_content_type(Path::new("foo/bar.json"), None),
  msg::MediaType::Json
  );
+ assert_eq!(
+ map_content_type(Path::new("foo/bar.wasm"), None),
+ msg::MediaType::Wasm
+ );
  assert_eq!(
  map_content_type(Path::new("foo/bar"), None),
  msg::MediaType::Unknown

cli/global_state.rs

@@ -3,6 +3,7 @@ use crate::compilers::CompiledModule;
 use crate::compilers::JsCompiler;
 use crate::compilers::JsonCompiler;
 use crate::compilers::TsCompiler;
+use crate::compilers::WasmCompiler;
 use crate::deno_dir;
 use crate::deno_error::permission_denied;
 use crate::file_fetcher::SourceFileFetcher;
@@ -45,6 +46,7 @@ pub struct GlobalState {
  pub js_compiler: JsCompiler,
  pub json_compiler: JsonCompiler,
  pub ts_compiler: TsCompiler,
+ pub wasm_compiler: WasmCompiler,
  pub lockfile: Option<Mutex<Lockfile>>,
 }
 
@@ -111,6 +113,7 @@ impl ThreadSafeGlobalState {
  ts_compiler,
  js_compiler: JsCompiler {},
  json_compiler: JsonCompiler {},
+ wasm_compiler: WasmCompiler::default(),
  lockfile,
  };
 
@@ -130,6 +133,9 @@ impl ThreadSafeGlobalState {
  .and_then(move |out| match out.media_type {
  msg::MediaType::Unknown => state1.js_compiler.compile_async(&out),
  msg::MediaType::Json => state1.json_compiler.compile_async(&out),
+ msg::MediaType::Wasm => {
+ state1.wasm_compiler.compile_async(state1.clone(), &out)
+ }
  msg::MediaType::TypeScript
  | msg::MediaType::TSX
  | msg::MediaType::JSX => {

cli/js/compiler.ts

@@ -28,7 +28,8 @@ enum MediaType {
  TypeScript = 2,
  TSX = 3,
  Json = 4,
- Unknown = 5
+ Wasm = 5,
+ Unknown = 6
 }
 
 // Warning! The values in this enum are duplicated in cli/msg.rs
@@ -44,8 +45,8 @@ enum CompilerRequestType {
 const console = new Console(core.print);
 window.console = console;
 window.workerMain = workerMain;
-function denoMain(): void {
- os.start(true, "TS");
+function denoMain(compilerType?: string): void {
+ os.start(true, compilerType || "TS");
 }
 window["denoMain"] = denoMain;
 
@@ -371,6 +372,9 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
  return ts.Extension.Tsx;
  case MediaType.Json:
  return ts.Extension.Json;
+ case MediaType.Wasm:
+ // Custom marker for Wasm type.
+ return ts.Extension.Js;
  case MediaType.Unknown:
  default:
  throw TypeError("Cannot resolve extension.");
@@ -724,3 +728,47 @@ window.compilerMain = function compilerMain(): void {
  workerClose();
  };
 };
+
+function base64ToUint8Array(data: string): Uint8Array {
+ const binString = window.atob(data);
+ const size = binString.length;
+ const bytes = new Uint8Array(size);
+ for (let i = 0; i < size; i++) {
+ bytes[i] = binString.charCodeAt(i);
+ }
+ return bytes;
+}
+
+window.wasmCompilerMain = function wasmCompilerMain(): void {
+ // workerMain should have already been called since a compiler is a worker.
+ window.onmessage = async ({
+ data: binary
+ }: {
+ data: string;
+ }): Promise<void> => {
+ const buffer = base64ToUint8Array(binary);
+ // @ts-ignore
+ const compiled = await WebAssembly.compile(buffer);
+
+ util.log(">>> WASM compile start");
+
+ const importList = Array.from(
+ // @ts-ignore
+ new Set(WebAssembly.Module.imports(compiled).map(({ module }) => module))
+ );
+ const exportList = Array.from(
+ // @ts-ignore
+ new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name))
+ );
+
+ postMessage({
+ importList,
+ exportList
+ });
+
+ util.log("<<< WASM compile end");
+
+ // The compiler isolate exits after a single message.
+ workerClose();
+ };
+};