Feat/dynamic rust plugin (#22)

* feat(rust-plugin): implement dynamic rust plugin CLI and loader

* chore: complete runtime

* chore: implement ResourceLoader

* chore: top level await implement

* chore: revert top level await impelment and leave it to future implemention

* fix: test and spell issue

* fix: windows resolve error

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+ "editor.formatOnSave": true,
+ "editor.tabSize": 2
+}

crates/compiler/examples/react.rs

@@ -0,0 +1,32 @@
+use farmfe_compiler::Compiler;
+use farmfe_core::{
+ config::{Config, RuntimeConfig},
+ relative_path::RelativePath,
+};
+
+fn main() {
+ let relative_root = RelativePath::new("examples/react");
+ let cwd = std::env::current_dir().unwrap();
+ let react_examples_root = relative_root.to_logical_path(cwd.clone());
+
+ let compiler = Compiler::new(
+ Config {
+ root: react_examples_root.to_string_lossy().to_string(),
+ runtime: RuntimeConfig {
+ path: cwd
+ .join("packages")
+ .join("runtime")
+ .join("src")
+ .join("index.ts")
+ .to_string_lossy()
+ .to_string(),
+ plugins: vec![],
+ },
+ ..Default::default()
+ },
+ vec![],
+ )
+ .unwrap();
+
+ compiler.compile().unwrap();
+}

crates/compiler/src/build/finalize_module.rs

@@ -0,0 +1,7 @@
+use std::sync::Arc;
+
+use farmfe_core::{context::CompilationContext, error::Result, module::ModuleId};
+
+pub fn finalize_module(module_id: &ModuleId, context: &Arc<CompilationContext>) -> Result<()> {
+ context.plugin_driver.finalize_module(module_id, context)
+}

crates/compiler/src/build/mod.rs

@@ -22,12 +22,14 @@ use farmfe_core::{
 
 use crate::{
  build::{
- analyze_deps::analyze_deps, load::load, parse::parse, resolve::resolve, transform::transform,
+ analyze_deps::analyze_deps, finalize_module::finalize_module, load::load, parse::parse,
+ resolve::resolve, transform::transform,
  },
  Compiler,
 };
 
 mod analyze_deps;
+mod finalize_module;
 mod load;
 mod parse;
 mod resolve;
@@ -212,6 +214,10 @@ impl Compiler {
  resolve_result.resolved_path, analyze_deps_result
  );
 
+ // ================ Finalize Module Start ===============
+ call_and_catch_error!(finalize_module, &module_id, &context);
+ // ================ Finalize Module End ===============
+
  // resolving dependencies recursively in the thread pool
  for (order, dep) in analyze_deps_result.into_iter().enumerate() {
  Self::build_module(

crates/compiler/src/generate/write_resources.rs

@@ -11,20 +11,20 @@ pub fn write_resources(context: &Arc<CompilationContext>) -> farmfe_core::error:
  .plugin_driver
  .write_resources(&mut *resources_map, context)?;
 
- println!("emit resources to disk");
+ // println!("emit resources to disk");
 
- resources_map.values().try_for_each(|resource| {
- // TODO determine the emit location by config
- if !resource.emitted {
- let root = PathBuf::from(context.config.root.as_str());
- let output_path = root.join("dist").join(&resource.name);
+ // resources_map.values().try_for_each(|resource| {
+ //  // TODO determine the emit location by config
+ //  if !resource.emitted {
+ //  let root = PathBuf::from(context.config.root.as_str());
+ //  let output_path = root.join("dist").join(&resource.name);
 
- std::fs::create_dir_all(output_path.parent().unwrap()).unwrap();
- std::fs::write(output_path, &resource.bytes).unwrap();
- }
+ //  std::fs::create_dir_all(output_path.parent().unwrap()).unwrap();
+ //  std::fs::write(output_path, &resource.bytes).unwrap();
+ //  }
 
- Ok::<(), CompilationError>(())
- })?;
+ //  Ok::<(), CompilationError>(())
+ // })?;
 
  Ok(())
 }

crates/compiler/src/lib.rs

@@ -61,7 +61,7 @@ impl Compiler {
  }
 
  pub fn update(&self, paths: Vec<String>) -> Result<UpdateOutput> {
- Ok(UpdateOutput {})
+ Ok(UpdateOutput::default())
  }
 
  pub fn context(&self) -> &Arc<CompilationContext> {

crates/compiler/src/update/mod.rs

@@ -1 +1,10 @@
-pub struct UpdateOutput {}
+use farmfe_core::module::ModuleId;
+
+/// The output after the updating process
+#[derive(Debug, Default)]
+pub struct UpdateOutput {
+ pub added_module_ids: Vec<ModuleId>,
+ pub updated_module_ids: Vec<ModuleId>,
+ pub removed_module_ids: Vec<ModuleId>,
+ pub resources: String,
+}

crates/core/Cargo.toml

@@ -24,6 +24,7 @@ petgraph = "0.6"
 bytecheck = "0.6.9"
 downcast-rs = "1.2"
 swc_ecma_ast = { version = "0.90.11", features = ["rkyv-impl"] }
+swc_ecma_parser = { version = "0.118.0" }
 swc_common = { version = "0.27.12", features = ["concurrent"] }
 swc_css_ast = { version = "0.108.0", features = ["rkyv-impl"] }
 swc_html_ast = { version = "0.24.7", features = ["rkyv-impl"] }

crates/core/src/config/mod.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
+use swc_ecma_ast::EsVersion;
+use swc_ecma_parser::{EsConfig, TsConfig};
 
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename = "camelCase", default)]
@@ -12,12 +14,13 @@ pub struct Config {
  pub resolve: ResolveConfig,
  pub external: Vec<String>,
  pub runtime: RuntimeConfig,
+ pub script: ScriptConfig,
 }
 
 impl Default for Config {
  fn default() -> Self {
  Self {
- input: HashMap::new(),
+ input: HashMap::from([("index".to_string(), "./index.html".to_string())]),
  root: std::env::current_dir()
  .unwrap()
  .to_string_lossy()
@@ -27,6 +30,7 @@ impl Default for Config {
  resolve: ResolveConfig::default(),
  external: vec![],
  runtime: Default::default(),
+ script: Default::default(),
  }
  }
 }
@@ -63,6 +67,18 @@ impl Default for Mode {
  }
 }
 
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+pub struct ScriptConfig {
+ pub target: EsVersion,
+ pub parser: ScriptParserConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+pub struct ScriptParserConfig {
+ pub es_config: EsConfig,
+ pub ts_config: TsConfig,
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename = "camelCase", default)]
 pub struct ResolveConfig {

crates/core/src/lib.rs

@@ -33,4 +33,5 @@ pub use serde_json;
 pub use swc_common;
 pub use swc_css_ast;
 pub use swc_ecma_ast;
+pub use swc_ecma_parser;
 pub use swc_html_ast;

crates/core/src/module/mod.rs

@@ -172,9 +172,13 @@ pub struct ScriptModuleMetaData {
 }
 
 #[cache_item]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum ModuleSystem {
  EsModule,
  CommonJs,
+ // Hybrid of commonjs and es-module
+ Hybrid,
+ Custom(String),
 }
 
 #[cache_item]

crates/core/src/plugin/mod.rs

@@ -3,7 +3,6 @@ use std::{any::Any, collections::HashMap, sync::Arc};
 use serde::{Deserialize, Serialize};
 
 use crate::{
- common::PackageJsonInfo,
  config::Config,
  context::CompilationContext,
  error::Result,
@@ -85,6 +84,14 @@ pub trait Plugin: Any + Send + Sync {
  Ok(None)
  }
 
+ fn finalize_module(
+ &self,
+ _module_id: &ModuleId,
+ _context: &Arc<CompilationContext>,
+ ) -> Result<Option<()>> {
+ Ok(None)
+ }
+
  /// The module graph should be constructed and finalized here
  fn build_end(&self, _context: &Arc<CompilationContext>) -> Result<Option<()>> {
  Ok(None)

crates/core/src/plugin/plugin_driver.rs

@@ -12,7 +12,9 @@ use crate::{
  config::Config,
  context::CompilationContext,
  error::Result,
- module::{module_graph::ModuleGraph, module_group::ModuleGroupMap, ModuleMetaData, ModuleType},
+ module::{
+ module_graph::ModuleGraph, module_group::ModuleGroupMap, ModuleId, ModuleMetaData, ModuleType,
+ },
  resource::{resource_pot::ResourcePot, resource_pot_graph::ResourcePotGraph, Resource},
  stats::Stats,
 };
@@ -139,6 +141,8 @@ impl PluginDriver {
 
  hook_serial!(analyze_deps, &mut PluginAnalyzeDepsHookParam);
 
+ hook_serial!(finalize_module, &ModuleId);
+
  hook_parallel!(build_end);
 
  hook_parallel!(generate_start);

crates/node/src/lib.rs

@@ -120,7 +120,10 @@ impl JsCompiler {
  let mut result = HashMap::new();
 
  for resource in resources.values() {
- result.insert(resource.name.clone(), resource.bytes.clone());
+ // only write expose non-emitted resource
+ if !resource.emitted {
+ result.insert(resource.name.clone(), resource.bytes.clone());
+ }
  }
 
  result

crates/plugin_html/tests/mod.rs

@@ -13,7 +13,7 @@ use farmfe_toolkit::testing_helpers::fixture;
 
 #[test]
 fn html_build_stage() {
- fixture("tests/fixtures/**/*.html", |file| {
+ fixture("tests/fixtures/**/*.html", |file, _| {
  let context = Arc::new(CompilationContext::new(Default::default(), vec![]).unwrap());
  let hook_context = PluginHookContext {
  caller: None,

crates/plugin_merge_modules/src/lib.rs

@@ -78,6 +78,11 @@ impl Plugin for FarmPluginMergeModules {
  for module_id in g.modules() {
  let module = module_graph.module_mut(module_id).unwrap();
 
+ // TODO remove this after the partial bundle is implemented
+ if module.module_type.is_script() {
+ module.module_type = ModuleType::Js;
+ }
+
  let resource_pot = if module_type_resource_pot_map.contains_key(&module.module_type) {
  let resource_pot = module_type_resource_pot_map
  .get_mut(&module.module_type)

crates/plugin_resolve/tests/mod.rs

@@ -6,7 +6,7 @@ use farmfe_toolkit::testing_helpers::fixture;
 fn resolve_relative_specifier() {
  fixture(
  "tests/fixtures/resolve-relative-specifier/**/index.*",
- |file| {
+ |file, _| {
  let resolver = Resolver::new(ResolveConfig::default());
  let cwd = file.parent().unwrap().to_path_buf();
 
@@ -25,7 +25,7 @@ fn resolve_relative_specifier() {
 fn resolve_node_modules_normal() {
  fixture(
  "tests/fixtures/resolve-node-modules/normal/index.ts",
- |file| {
+ |file, _| {
  let cwd = file.parent().unwrap().to_path_buf();
  let resolver = Resolver::new(ResolveConfig::default());
 

crates/plugin_runtime/src/js-runtime/resource-wrapper.js

@@ -1,4 +1,4 @@
-(function (modules /* TODO url */) {
+(function (modules) {
  for (var key in modules) {
  globalThis.__acquire_farm_module_system__().register(key, modules[key]);
  }