Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: native plugin init #1691

Merged
merged 10 commits into from
Dec 19, 2024
Merged

feat: native plugin init #1691

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1c4869c
feat: init rust plugin
Jinbao1001 Nov 13, 2024
c315018
chore: merge
Jinbao1001 Nov 13, 2024
9b01d6c
chore: delete print
Jinbao1001 Nov 13, 2024
5cde533
Rename cargo.toml to Cargo.toml
Jinbao1001 Nov 13, 2024
a63f42e
chore: update
Jinbao1001 Dec 16, 2024
7d315ba
Merge remote-tracking branch 'origin/master' into feat/rust-plugin
Jinbao1001 Dec 16, 2024
c175fc1
fix: type
Jinbao1001 Dec 17, 2024
9dedf58
fix: plugin options string
Jinbao1001 Dec 18, 2024
8848886
Merge remote-tracking branch 'origin/master' into feat/rust-plugin
Jinbao1001 Dec 18, 2024
621903d
fix: cfg test
Jinbao1001 Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 218 additions & 26 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ resolver = "2"
anyhow = "1.0.71"
cached = "0.46.1"
clap = "4.3.11"
mimalloc-rust = { version = "=0.2.1" }
oneshot = "0.1.8"
regex = "1.9.3"
serde = "1.0.211"
serde_json = "1.0.132"
swc_core = { version = "0.101.4", default-features = false }
swc_malloc = "1.0.0"
tikv-jemallocator = { version = "=0.5.4", features = ["disable_initial_exec_tls"] }
xusd320 marked this conversation as resolved.
Show resolved Hide resolved

[profile.release]
Expand Down
7 changes: 1 addition & 6 deletions crates/binding/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,7 @@ napi-derive = "2.16.12"
oneshot = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

[target.'cfg(not(target_os = "linux"))'.dependencies]
mimalloc-rust = { workspace = true }

[target.'cfg(all(target_os = "linux", target_env = "gnu", any(target_arch = "x86_64", target_arch = "aarch64")))'.dependencies]
tikv-jemallocator = { workspace = true }
swc_malloc = { workspace = true }

[build-dependencies]
napi-build = "2.0.1"
15 changes: 3 additions & 12 deletions crates/binding/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#![deny(clippy::all)]

extern crate swc_malloc;

use std::sync::{Arc, Once};

use js_hook::{JsHooks, TsFnHooks};
Expand All @@ -18,18 +20,6 @@ mod js_hook;
mod js_plugin;
mod threadsafe_function;

#[cfg(not(target_os = "linux"))]
#[global_allocator]
static GLOBAL: mimalloc_rust::GlobalMiMalloc = mimalloc_rust::GlobalMiMalloc;

#[cfg(all(
target_os = "linux",
target_env = "gnu",
any(target_arch = "x86_64", target_arch = "aarch64")
))]
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

static LOG_INIT: Once = Once::new();

#[napi(object)]
Expand Down Expand Up @@ -165,6 +155,7 @@ pub struct BuildParams {
};
experimental?: {
webpackSyntaxValidate?: string[];
rustPlugins?: Array<[string, any]>;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

建议加强类型定义和文档说明

当前的 rustPlugins 类型定义过于宽松：

  1. 使用 any 类型不利于类型安全
  2. 缺少对插件配置结构的文档说明

建议修改为：

- rustPlugins?: Array<[string, any]>;
+ rustPlugins?: Array<{
+   // 插件的路径
+   path: string;
+   // 插件的配置选项
+   options?: {
+     [key: string]: string | number | boolean;
+   };
+ }>;

同时建议添加相关文档说明插件的配置格式和用法。

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
rustPlugins?: Array<[string, any]>;
rustPlugins?: Array<{
// 插件的路径
path: string;
// 插件的配置选项
options?: {
[key: string]: string | number | boolean;
};
}>;

};
watch?: {
ignoredPaths?: string[];
Expand Down
15 changes: 8 additions & 7 deletions crates/mako/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ oxc_resolver = { version = "=1.7.0", features = ["package_json_raw_json_api"
percent-encoding = { version = "2.3.1" }
serde = { workspace = true }
serde_json = { workspace = true }
swc_malloc = { workspace = true }
url = { version = "2.5.0" }

swc_core = { workspace = true, features = [
Expand All @@ -47,6 +48,7 @@ swc_core = { workspace = true, features = [
"ecma_minifier",
"ecma_minifier_concurrent",
"ecma_parser",
"ecma_plugin_transform",
"ecma_preset_env",
"ecma_quote",
"ecma_transforms_compat",
Expand All @@ -57,10 +59,11 @@ swc_core = { workspace = true, features = [
"ecma_transforms_typescript",
"ecma_utils",
"ecma_visit_path",
"swc_common",
"swc_ecma_quote_macros",
] }

swc_emotion = "0.72.19"
swc_emotion = "=0.72.22"
swc_error_reporters = "0.21.0"
swc_node_comments = "0.24.0"

Expand All @@ -81,6 +84,7 @@ hyper-staticfile = "0.9.6"
hyper-tungstenite = "0.10.0"
indexmap = "2.0.0"
indicatif = "0.17.8"
libloading = "0.8"
md5 = "0.7.0"
mdxjs = "0.2.6"
mime_guess = "2.0.4"
Expand All @@ -90,15 +94,18 @@ parking_lot = { version = "0.12", features = ["nightly"] }
path-clean = "1.0.1"
pathdiff = "0.2.1"
petgraph = "0.6.3"
proc-macro2 = "1"
puffin = { version = "0.16.0", optional = true }
puffin_egui = { version = "0.22.0", optional = true }
quote = "1"
rayon = "1.7.0"
regex = { workspace = true }
sailfish = "0.8.3"
semver = "1.0.23"
serde-xml-rs = "0.6.0"
serde_yaml = "0.9.22"
svgr-rs = { path = "../svgr-rs" }
syn = { version = "2", features = ["full"] }
thiserror = "1.0.43"
tokio = { version = "1", features = ["rt-multi-thread", "sync"] }
tokio-tungstenite = "0.19.0"
Expand All @@ -108,12 +115,6 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
tungstenite = "0.19.0"
twox-hash = "1.6.3"

[target.'cfg(not(target_os = "linux"))'.dependencies]
mimalloc-rust = { workspace = true }

[target.'cfg(all(target_os = "linux", target_env = "gnu", any(target_arch = "x86_64", target_arch = "aarch64")))'.dependencies]
tikv-jemallocator = { workspace = true }

[dev-dependencies]
insta = { version = "1.30.0", features = ["yaml"] }
maplit = "1.0.2"
Expand Down
2 changes: 1 addition & 1 deletion crates/mako/src/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ pub(crate) mod js_ast;
pub(crate) mod sourcemap;
#[cfg(test)]
pub mod tests;
pub(crate) mod utils;
pub mod utils;

pub const DUMMY_CTXT: SyntaxContext = SyntaxContext::empty();
20 changes: 16 additions & 4 deletions crates/mako/src/compiler.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use std::time::Instant;

use anyhow::{anyhow, Error, Result};
use colored::Colorize;
use libloading::Library;
use regex::Regex;
use swc_core::common::sync::Lrc;
use swc_core::common::{Globals, SourceMap, DUMMY_SP};
Expand Down Expand Up @@ -220,7 +221,7 @@ pub struct Compiler {

impl Compiler {
pub fn new(
config: Config,
mut config: Config,
stormslowly marked this conversation as resolved.
Show resolved Hide resolved
root: PathBuf,
args: Args,
extra_plugins: Option<Vec<Arc<dyn Plugin>>>,
Expand All @@ -238,6 +239,19 @@ impl Compiler {
if let Some(extra_plugins) = extra_plugins {
plugins.extend(extra_plugins);
}

let mut external_plugins: Vec<Arc<dyn Plugin>> = vec![];
unsafe {
xusd320 marked this conversation as resolved.
Show resolved Hide resolved
for rust_plugin in config.experimental.rust_plugins.clone() {
let lib = Arc::new(Library::new(rust_plugin.path)?);
let plugin_create_fn: libloading::Symbol<
unsafe extern "C" fn(option: String) -> Arc<dyn Plugin>,
> = lib.get(b"_plugin_create").unwrap();
let plugin = plugin_create_fn(rust_plugin.options);
external_plugins.push(plugin);
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

需要加强动态库加载的安全性和错误处理

当前实现存在以下问题：

  1. 缺少对加载库的有效性验证
  2. 错误处理不够完善
  3. 未对插件接口进行版本检查

建议重构为：

+ fn load_rust_plugin(plugin_path: String, options: String) -> Result<Arc<dyn Plugin>> {
+     // 验证插件文件是否存在且具有正确的格式
+     if !std::path::Path::new(&plugin_path).exists() {
+         return Err(anyhow!("Plugin file not found: {}", plugin_path));
+     }
+     
+     unsafe {
+         let lib = Library::new(&plugin_path)
+             .map_err(|e| anyhow!("Failed to load plugin {}: {}", plugin_path, e))?;
+         
+         // 验证插件版本兼容性
+         let version_fn: libloading::Symbol<unsafe extern "C" fn() -> u32> = 
+             lib.get(b"_plugin_version")
+             .map_err(|_| anyhow!("Plugin {} is missing version information", plugin_path))?;
+         
+         let version = version_fn();
+         if version != PLUGIN_API_VERSION {
+             return Err(anyhow!("Plugin {} version mismatch", plugin_path));
+         }
+         
+         // 加载插件创建函数
+         let create_fn: libloading::Symbol<unsafe extern "C" fn(String) -> Arc<dyn Plugin>> = 
+             lib.get(b"_plugin_create")
+             .map_err(|_| anyhow!("Plugin {} is invalid", plugin_path))?;
+         
+         Ok(create_fn(options))
+     }
+ }

  let mut external_plugins: Vec<Arc<dyn Plugin>> = vec![];
  for rust_plugin in config.experimental.rust_plugins.clone() {
-     unsafe {
-         let lib = Arc::new(Library::new(rust_plugin.path)?);
-         let plugin_create_fn: libloading::Symbol<
-             unsafe extern "C" fn(option: String) -> Arc<dyn Plugin>,
-         > = lib.get(b"_plugin_create").unwrap();
-         let plugin = plugin_create_fn(rust_plugin.options);
-         external_plugins.push(plugin);
-     }
+     match load_rust_plugin(rust_plugin.path, rust_plugin.options) {
+         Ok(plugin) => external_plugins.push(plugin),
+         Err(e) => {
+             eprintln!("Warning: Failed to load plugin: {}", e);
+             continue;
+         }
+     }
  }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
unsafe {
for rust_plugin in config.experimental.rust_plugins.clone() {
let lib = Arc::new(Library::new(rust_plugin.path)?);
let plugin_create_fn: libloading::Symbol<
unsafe extern "C" fn(option: String) -> Arc<dyn Plugin>,
> = lib.get(b"_plugin_create").unwrap();
let plugin = plugin_create_fn(rust_plugin.options);
external_plugins.push(plugin);
}
}
fn load_rust_plugin(plugin_path: String, options: String) -> Result<Arc<dyn Plugin>> {
// 验证插件文件是否存在且具有正确的格式
if !std::path::Path::new(&plugin_path).exists() {
return Err(anyhow!("Plugin file not found: {}", plugin_path));
}
unsafe {
let lib = Library::new(&plugin_path)
.map_err(|e| anyhow!("Failed to load plugin {}: {}", plugin_path, e))?;
// 验证插件版本兼容性
let version_fn: libloading::Symbol<unsafe extern "C" fn() -> u32> =
lib.get(b"_plugin_version")
.map_err(|_| anyhow!("Plugin {} is missing version information", plugin_path))?;
let version = version_fn();
if version != PLUGIN_API_VERSION {
return Err(anyhow!("Plugin {} version mismatch", plugin_path));
}
// 加载插件创建函数
let create_fn: libloading::Symbol<unsafe extern "C" fn(String) -> Arc<dyn Plugin>> =
lib.get(b"_plugin_create")
.map_err(|_| anyhow!("Plugin {} is invalid", plugin_path))?;
Ok(create_fn(options))
}
}
let mut external_plugins: Vec<Arc<dyn Plugin>> = vec![];
for rust_plugin in config.experimental.rust_plugins.clone() {
match load_rust_plugin(rust_plugin.path, rust_plugin.options) {
Ok(plugin) => external_plugins.push(plugin),
Err(e) => {
eprintln!("Warning: Failed to load plugin: {}", e);
continue;
}
}
}


let builtin_plugins: Vec<Arc<dyn Plugin>> = vec![
// features
Arc::new(plugins::manifest::ManifestPlugin {}),
Expand All @@ -254,10 +268,9 @@ impl Compiler {
Arc::new(plugins::tree_shaking::FarmTreeShake {}),
Arc::new(plugins::detect_circular_dependence::LoopDetector {}),
];
plugins.extend(external_plugins);
plugins.extend(builtin_plugins);

let mut config = config;

if let Some(progress) = &config.progress {
plugins.push(Arc::new(plugins::progress::ProgressPlugin::new(
plugins::progress::ProgressPluginOptions {
Expand Down Expand Up @@ -480,7 +493,6 @@ impl Compiler {
let mg = self.context.module_graph.read().unwrap();
cg.full_hash(&mg)
}

fn clean_dist(&self) -> Result<()> {
// compiler 前清除 dist，如果后续 dev 环境不在 output_path 里，需要再补上 dev 的逻辑
let output_path = &self.context.config.output.path;
Expand Down
24 changes: 23 additions & 1 deletion crates/mako/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ pub use resolve::ResolveConfig;
pub use rsc_client::{deserialize_rsc_client, LogServerComponent, RscClientConfig};
pub use rsc_server::{deserialize_rsc_server, RscServerConfig};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_json::{json, Value};
pub use stats::{deserialize_stats, StatsConfig};
use thiserror::Error;
pub use transform_import::{TransformImportConfig, TransformImportStyle};
Expand Down Expand Up @@ -232,14 +232,27 @@ impl Config {
) -> Result<Self> {
let abs_config_file = root.join(CONFIG_FILE);
let abs_config_file = abs_config_file.to_str().unwrap();
let mut overrides_json: Option<String> = None;
let c = config::Config::builder();
// default config
let c = c.add_source(config::File::from_str(
DEFAULT_CONFIG,
config::FileFormat::Json5,
));

// default config from args
let c = if let Some(default_config) = default_config {
let result: Result<Value, serde_json::Error> = serde_json::from_str(default_config);
if let Ok(config) = result {
if let Some(experimental) = config.get("experimental") {
overrides_json = Some(
serde_json::to_string(&json!({
"experimental": experimental
}))
.unwrap(),
);
}
};
c.add_source(config::File::from_str(
default_config,
config::FileFormat::Json5,
Expand All @@ -260,6 +273,15 @@ impl Config {
} else {
c
};
// overrides config
let c = if let Some(overrides) = overrides_json {
c.add_source(config::File::from_str(
overrides.as_str(),
config::FileFormat::Json5,
))
} else {
c
};

let c = c.build()?;
let mut ret = c.try_deserialize::<Config>();
Expand Down
7 changes: 7 additions & 0 deletions crates/mako/src/config/experimental.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ use serde::{Deserialize, Serialize};

use crate::create_deserialize_fn;

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct RustPlugin {
pub path: String,
pub options: String,
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

建议改进 RustPlugin 结构体的设计

当前实现存在以下需要改进的地方：

  1. 缺少结构体和字段的文档注释，建议添加以说明用途和预期值
  2. options 字段使用 String 类型可能过于简单，建议考虑使用更结构化的类型（如 serde_json::Value）来支持更复杂的配置

建议按如下方式修改：

 #[derive(Deserialize, Serialize, Debug, Clone)]
+/// 用于配置 Rust 原生插件的结构体
 pub struct RustPlugin {
+    /// 插件的文件路径
     pub path: String,
+    /// 插件的配置选项，支持 JSON 格式
-    pub options: String,
+    pub options: serde_json::Value,
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct RustPlugin {
pub path: String,
pub options: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
/// 用于配置 Rust 原生插件的结构体
pub struct RustPlugin {
/// 插件的文件路径
pub path: String,
/// 插件的配置选项，支持 JSON 格式
pub options: serde_json::Value,
}


#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ExperimentalConfig {
Expand All @@ -12,6 +18,7 @@ pub struct ExperimentalConfig {
pub magic_comment: bool,
#[serde(deserialize_with = "deserialize_detect_loop")]
pub detect_circular_dependence: Option<DetectCircularDependence>,
pub rust_plugins: Vec<RustPlugin>,
pub central_ensure: bool,
}

Expand Down
1 change: 1 addition & 0 deletions crates/mako/src/config/mako.config.default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
"rscClient": false,
"experimental": {
"webpackSyntaxValidate": [],
"rustPlugins": [],
"requireContext": true,
"ignoreNonLiteralRequire": false,
"magicComment": true,
Expand Down
4 changes: 3 additions & 1 deletion crates/mako/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ pub mod config;
pub mod dev;
mod features;
mod generate;
mod module;
pub mod module;
mod module_graph;
pub mod plugin;
mod plugins;
Expand All @@ -20,6 +20,8 @@ pub mod stats;
pub mod utils;
mod visitors;

pub use {swc_core, swc_malloc};

#[macro_export]
macro_rules! mako_profile_scope {
($id:expr) => {
Expand Down
17 changes: 5 additions & 12 deletions crates/mako/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#![feature(box_patterns)]
#![feature(let_chains)]

extern crate swc_malloc;

use std::sync::Arc;

use anyhow::{anyhow, Result};
Expand All @@ -15,18 +17,6 @@ use mako::utils::tokio_runtime;
use mako::{cli, config};
use tracing::debug;

#[cfg(not(target_os = "linux"))]
#[global_allocator]
static GLOBAL: mimalloc_rust::GlobalMiMalloc = mimalloc_rust::GlobalMiMalloc;

#[cfg(all(
target_os = "linux",
target_env = "gnu",
any(target_arch = "x86_64", target_arch = "aarch64")
))]
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

fn main() -> Result<()> {
let fut = async { run().await };

Expand Down Expand Up @@ -88,10 +78,13 @@ async fn run() -> Result<()> {

#[cfg(not(feature = "profile"))]
{
print!("开始执行compiler");
if let Err(e) = compiler.compile() {
print!("进到error里了");
eprintln!("{}", e);
std::process::exit(1);
}
print!("compiler执行完毕");
if cli.watch {
let d = dev::DevServer::new(root.clone(), compiler);
// TODO: when in Dev Mode, Dev Server should start asap, and provider a loading while in first compiling
Expand Down
13 changes: 13 additions & 0 deletions crates/mako_plugin_macro/cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[package]
name = "mako_plugin_macro"
version = "0.0.1"
edition = "2021"
license = "MIT"

[dependencies]
proc-macro2 = "1"
quote = "1"
syn = { version = "2", features = ["full"] }

[lib]
proc-macro = true
Loading
Loading