diff --git a/crates/moon/src/cli.rs b/crates/moon/src/cli.rs index 6dd2bbc6..0c6398ab 100644 --- a/crates/moon/src/cli.rs +++ b/crates/moon/src/cli.rs @@ -115,6 +115,7 @@ pub enum MoonBuildSubcommands { // Misc Coverage(CoverageSubcommand), GenerateBuildMatrix(GenerateBuildMatrix), + /// Upgrade toolchains Upgrade(UpgradeSubcommand), ShellCompletion(ShellCompSubCommand), diff --git a/crates/moon/src/cli/tool.rs b/crates/moon/src/cli/tool.rs index f35aed63..772a0e88 100644 --- a/crates/moon/src/cli/tool.rs +++ b/crates/moon/src/cli/tool.rs @@ -16,8 +16,12 @@ // // For inquiries, you can contact us via e-mail at jichuruanjian@idea.edu.cn. -mod format_and_diff; +use std::path::PathBuf; +pub mod embed; +pub mod format_and_diff; + +use embed::*; use format_and_diff::*; #[derive(Debug, clap::Parser)] @@ -29,10 +33,20 @@ pub struct ToolSubcommand { #[derive(Debug, clap::Parser)] pub enum ToolSubcommands { FormatAndDiff(FormatAndDiffSubcommand), + Embed(Embed), +} + +#[derive(Debug, clap::Parser)] +pub struct FormatAndWriteSubcommand { + #[clap(long)] + old: PathBuf, + #[clap(long)] + new: PathBuf, } pub fn run_tool(cmd: ToolSubcommand) -> anyhow::Result { match cmd.subcommand { ToolSubcommands::FormatAndDiff(subcmd) => run_format_and_diff(subcmd), + ToolSubcommands::Embed(subcmd) => run_embed(subcmd), } } diff --git a/crates/moon/src/cli/tool/embed.rs b/crates/moon/src/cli/tool/embed.rs new file mode 100644 index 00000000..14bd3a6f --- /dev/null +++ b/crates/moon/src/cli/tool/embed.rs @@ -0,0 +1,86 @@ +// moon: The build system and package manager for MoonBit. +// Copyright (C) 2024 International Digital Economy Academy +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// +// For inquiries, you can contact us via e-mail at jichuruanjian@idea.edu.cn. + +use std::path::PathBuf; + +use anyhow::Context; + +#[derive(Debug, clap::Parser)] +pub struct Embed { + #[clap(long, conflicts_with = "text")] + binary: bool, + #[clap(long, conflicts_with = "binary")] + text: bool, + #[clap(long, short)] + input: PathBuf, + #[clap(long, short)] + output: PathBuf, + #[clap(long)] + name: Option, + #[clap(long)] + timestamp: bool, +} + +pub fn run_embed_text(cmd: Embed) -> anyhow::Result { + let input = std::fs::read_to_string(&cmd.input)?; + let name = cmd.name.unwrap_or_else(|| "resource".to_string()); + let mut content = format!( + "// Generated by `moon tool embed --text`{}, do not edit.\n\n\ + let {} : String =\n", + if cmd.timestamp { + format!(" on {}", chrono::Local::now().format("%Y-%m-%d %H:%M:%S")) + } else { + String::new() + }, + name + ); + for line in input.split('\n') { + content.push_str(&format!(" #|{}\n", line)); + } + std::fs::write(cmd.output, content).context("write output file")?; + Ok(0) +} + +pub fn run_embed_bin(cmd: Embed) -> anyhow::Result { + let input = std::fs::read(&cmd.input)?; + let name = cmd.name.unwrap_or_else(|| "resource".to_string()); + let mut content = format!( + "// Generated by `moon tool embed --binary`{}, do not edit.\n\n\ + let {} : Bytes = b\"", + if cmd.timestamp { + format!(" on {}", chrono::Local::now().format("%Y-%m-%d %H:%M:%S")) + } else { + String::new() + }, + name + ); + for byte in input.iter() { + content.push_str(&format!("\\x{:02x}", byte)); + } + content.push_str("\"\n"); + std::fs::write(cmd.output, content).context("write output file")?; + Ok(0) +} + +pub fn run_embed(cmd: Embed) -> anyhow::Result { + if cmd.binary { + run_embed_bin(cmd) + } else { + run_embed_text(cmd) + } +} diff --git a/crates/moon/tests/test_cases/pre_build.in/.gitignore b/crates/moon/tests/test_cases/pre_build.in/.gitignore new file mode 100644 index 00000000..b1283a74 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/.gitignore @@ -0,0 +1,2 @@ +target/ +.mooncakes/ diff --git a/crates/moon/tests/test_cases/pre_build.in/README.md b/crates/moon/tests/test_cases/pre_build.in/README.md new file mode 100644 index 00000000..ae00983f --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/README.md @@ -0,0 +1 @@ +# username/hello \ No newline at end of file diff --git a/crates/moon/tests/test_cases/pre_build.in/moon.mod.json b/crates/moon/tests/test_cases/pre_build.in/moon.mod.json new file mode 100644 index 00000000..9e55a905 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/moon.mod.json @@ -0,0 +1,10 @@ +{ + "name": "username/hello", + "version": "0.1.0", + "readme": "README.md", + "repository": "", + "license": "", + "keywords": [], + "description": "", + "source": "src" +} \ No newline at end of file diff --git a/crates/moon/tests/test_cases/pre_build.in/moon.test.ignore b/crates/moon/tests/test_cases/pre_build.in/moon.test.ignore new file mode 100644 index 00000000..bda55c87 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/moon.test.ignore @@ -0,0 +1,26 @@ + $ moon check + + Executed 3 pre-build tasks, now up to date + Finished. moon: ran 2 tasks, now up to date + + $ xcat src/lib/a.mbt + // Generated by `moon tool embed --text` do not edit. + + let resource : String = + #|hello, + #|world + #| + + $ xcat src/lib/b.mbt + // Generated by `moon tool embed --binary` do not edit. + + let _b : Bytes = b"/x6d/x6f/x6f/x6e/x0a/x67/x65/x6e/x65/x72/x61/x74/x65/x0a" + + $ xcat src/lib/c.mbt + // Generated by `moon tool embed --text` do not edit. + + let _c : String = + #|hello, + #|world + #| + diff --git a/crates/moon/tests/test_cases/pre_build.in/src/lib/a.txt b/crates/moon/tests/test_cases/pre_build.in/src/lib/a.txt new file mode 100644 index 00000000..d5ff25e7 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/lib/a.txt @@ -0,0 +1,2 @@ +hello, +world diff --git a/crates/moon/tests/test_cases/pre_build.in/src/lib/b.txt b/crates/moon/tests/test_cases/pre_build.in/src/lib/b.txt new file mode 100644 index 00000000..18e5c770 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/lib/b.txt @@ -0,0 +1,2 @@ +moon +generate diff --git a/crates/moon/tests/test_cases/pre_build.in/src/lib/c.txt b/crates/moon/tests/test_cases/pre_build.in/src/lib/c.txt new file mode 100644 index 00000000..d5ff25e7 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/lib/c.txt @@ -0,0 +1,2 @@ +hello, +world diff --git a/crates/moon/tests/test_cases/pre_build.in/src/lib/hello.mbt b/crates/moon/tests/test_cases/pre_build.in/src/lib/hello.mbt new file mode 100644 index 00000000..9012592a --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/lib/hello.mbt @@ -0,0 +1,3 @@ +pub fn hello() -> String { + "Hello, world!" +} diff --git a/crates/moon/tests/test_cases/pre_build.in/src/lib/moon.pkg.json b/crates/moon/tests/test_cases/pre_build.in/src/lib/moon.pkg.json new file mode 100644 index 00000000..59627050 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/lib/moon.pkg.json @@ -0,0 +1,19 @@ +{ + "pre-build": [ + { + "input": "a.txt", + "output": "a.mbt", + "command": ":embed --text -i ${input} -o ${output}" + }, + { + "input": ["b.txt"], + "output": "b.mbt", + "command": ":embed --binary -i ${input} -o ${output} --name _b" + }, + { + "input": "c.txt", + "output": "c.mbt", + "command": ":embed --text -i ${input} -o ${output} --name _c" + } + ] +} diff --git a/crates/moon/tests/test_cases/pre_build.in/src/main/main.mbt b/crates/moon/tests/test_cases/pre_build.in/src/main/main.mbt new file mode 100644 index 00000000..d25cb935 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/main/main.mbt @@ -0,0 +1,3 @@ +fn main { + println(@lib.hello()) +} diff --git a/crates/moon/tests/test_cases/pre_build.in/src/main/moon.pkg.json b/crates/moon/tests/test_cases/pre_build.in/src/main/moon.pkg.json new file mode 100644 index 00000000..48783525 --- /dev/null +++ b/crates/moon/tests/test_cases/pre_build.in/src/main/moon.pkg.json @@ -0,0 +1,6 @@ +{ + "is-main": true, + "import": [ + "username/hello/lib" + ] +} \ No newline at end of file diff --git a/crates/moonbuild/src/bench.rs b/crates/moonbuild/src/bench.rs index a29718b9..ce061bd8 100644 --- a/crates/moonbuild/src/bench.rs +++ b/crates/moonbuild/src/bench.rs @@ -109,6 +109,7 @@ pub fn f() -> Unit {{ warn_list: None, alert_list: None, targets: None, + pre_build: None, }; moonutil::common::write_package_json_to_file(&pkg, &moon_pkg).unwrap(); } @@ -190,6 +191,7 @@ pub fn write(config: &Config, base_dir: &Path) { warn_list: None, alert_list: None, targets: None, + pre_build: None, }; moonutil::common::write_package_json_to_file(&pkg, &base_dir.join("main").join(MOON_PKG_JSON)) diff --git a/crates/moonbuild/src/entry.rs b/crates/moonbuild/src/entry.rs index b85c15bc..5999c5e9 100644 --- a/crates/moonbuild/src/entry.rs +++ b/crates/moonbuild/src/entry.rs @@ -87,6 +87,59 @@ fn render_result(result: Option, quiet: bool, mode: &str) -> anyhow::Resu } } +pub fn n2_simple_run_interface( + state: n2::load::State, + moonbuild_opt: &MoonbuildOpt, +) -> anyhow::Result> { + let logger = Arc::new(Mutex::new(vec![])); + let use_fancy = terminal::use_fancy(); + + let catcher = Arc::clone(&logger); + let output_json = moonbuild_opt.output_json; + let render_and_catch = move |output: &str| { + output + .split('\n') + .filter(|it| !it.is_empty()) + .for_each(|content| { + catcher.lock().unwrap().push(content.to_owned()); + if output_json { + println!("{content}"); + } else { + moonutil::render::MooncDiagnostic::render(content, use_fancy); + } + }); + }; + + // TODO: generate build graph for pre_build? + + let mut progress = create_progress_console(Some(Box::new(render_and_catch))); + let options = work::Options { + parallelism: default_parallelism()?, + failures_left: Some(10), + explain: false, + adopt: false, + }; + let mut work = work::Work::new( + state.graph, + state.hashes, + state.db, + &options, + progress.as_mut(), + state.pools, + ); + + if !state.default.is_empty() { + for target in state.default { + work.want_file(target)?; + } + } else { + return Ok(Some(0)); + } + + let res = trace::scope("work.run", || work.run())?; + Ok(res) +} + pub fn n2_run_interface( state: n2::load::State, moonbuild_opt: &MoonbuildOpt, @@ -231,11 +284,39 @@ fn vis_build_graph(state: &State, moonbuild_opt: &MoonbuildOpt) { eprintln!("generated build graph: {}", path.display()); } +fn run_moon_generate(moonbuild_opt: &MoonbuildOpt, module: &ModuleDB) -> anyhow::Result { + let generate_state = crate::generate::load_moon_generate(moonbuild_opt, module)?; + let generate_result = n2_simple_run_interface(generate_state, moonbuild_opt)?; + render_generate_result(generate_result, moonbuild_opt.quiet)?; + Ok(0) +} + +fn render_generate_result(result: Option, quiet: bool) -> anyhow::Result { + match result { + None => { + anyhow::bail!(format!("failed when execute generate")); + } + Some(0) => Ok(0), + Some(n) => { + if !quiet { + eprintln!( + "Executed {} pre-build task{}, now up to date", + n, + if n == 1 { "" } else { "s" } + ); + } + Ok(0) + } + } +} + pub fn run_check( moonc_opt: &MooncOpt, moonbuild_opt: &MoonbuildOpt, module: &ModuleDB, ) -> anyhow::Result { + run_moon_generate(moonbuild_opt, module)?; + let state = trace::scope("moonbit::check::read", || { crate::check::normal::load_moon_proj(module, moonc_opt, moonbuild_opt) })?; @@ -256,6 +337,8 @@ pub fn run_build( moonbuild_opt: &MoonbuildOpt, module: &ModuleDB, ) -> anyhow::Result { + run_moon_generate(moonbuild_opt, module)?; + let state = trace::scope("moonbit::build::read", || { crate::build::load_moon_proj(module, moonc_opt, moonbuild_opt) })?; @@ -455,6 +538,8 @@ pub fn run_test( auto_update: bool, module: ModuleDB, ) -> anyhow::Result>> { + run_moon_generate(&moonbuild_opt, &module)?; + let moonc_opt = Arc::new(moonc_opt); let moonbuild_opt = Arc::new(moonbuild_opt); let module = Arc::new(module); @@ -928,6 +1013,8 @@ pub fn run_bundle( moonbuild_opt: &MoonbuildOpt, moonc_opt: &MooncOpt, ) -> anyhow::Result { + run_moon_generate(moonbuild_opt, module)?; + let state = crate::bundle::load_moon_proj(module, moonc_opt, moonbuild_opt)?; let result = n2_run_interface(state, moonbuild_opt)?; match result { diff --git a/crates/moonbuild/src/generate.rs b/crates/moonbuild/src/generate.rs new file mode 100644 index 00000000..cce9fe6e --- /dev/null +++ b/crates/moonbuild/src/generate.rs @@ -0,0 +1,127 @@ +// moon: The build system and package manager for MoonBit. +// Copyright (C) 2024 International Digital Economy Academy +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// +// For inquiries, you can contact us via e-mail at jichuruanjian@idea.edu.cn. + +use std::path::PathBuf; +use std::rc::Rc; + +use moonutil::common::MoonbuildOpt; +use moonutil::module::ModuleDB; +use moonutil::package::StringOrArray; +use n2::graph::{self as n2graph, Build, BuildIns, BuildOuts, FileId, FileLoc}; +use n2::load::State; +use n2::smallmap::SmallMap; + +pub fn load_moon_generate( + moonbuild_opt: &MoonbuildOpt, + module: &ModuleDB, +) -> anyhow::Result { + let mut graph = n2graph::Graph::default(); + let mut defaults: Vec = vec![]; + + for (_, pkg) in module.packages.iter() { + if pkg.is_third_party { + continue; + } + if let Some(generate) = &pkg.pre_build { + for rule in generate { + let cwd = &pkg.root_path; + let input = &rule.input; + let output = &rule.output; + let command = &rule.command; + let inputs = match input { + StringOrArray::String(s) => { + vec![cwd.join(s)] + } + StringOrArray::Array(arr) => { + arr.iter().map(|s| cwd.join(s)).collect::>() + } + }; + let inputs = inputs + .iter() + .map(|p| p.display().to_string()) + .collect::>(); + + let inputs_ids = inputs + .iter() + .map(|f| graph.files.id_from_canonical(f.into())) + .collect::>(); + + let outputs = match output { + StringOrArray::String(s) => vec![cwd.join(s)], + StringOrArray::Array(arr) => { + arr.iter().map(|s| cwd.join(s)).collect::>() + } + }; + let outputs = outputs + .iter() + .map(|p| p.display().to_string()) + .collect::>(); + let outputs_ids = outputs + .iter() + .map(|f| graph.files.id_from_canonical(f.into())) + .collect::>(); + + let ins = BuildIns { + explicit: inputs_ids.len(), + ids: inputs_ids, + implicit: 0, + order_only: 0, + }; + for o in outputs_ids.iter() { + defaults.push(*o); + } + let outs = BuildOuts { + explicit: outputs_ids.len(), + ids: outputs_ids, + }; + + let loc = FileLoc { + filename: Rc::new(PathBuf::from("generate")), + line: 0, + }; + + let mut build = Build::new(loc, ins, outs); + let command = if command.starts_with(":embed") { + command.replacen(":embed", "moon tool embed", 1).to_string() + } else { + command.to_string() + }; + let command = command + .replace("${input}", &inputs.join(" ")) + .replace("${output}", &outputs.join(" ")); + build.cmdline = Some(command.clone()); + graph.add_build(build).unwrap(); + } + } + } + + let mut hashed = n2graph::Hashes::default(); + let common = moonbuild_opt.raw_target_dir.join("common"); + if !common.exists() { + std::fs::create_dir_all(&common)?; + } + let n2_db_path = common.join("generate.db"); + let db = n2::db::open(&n2_db_path, &mut graph, &mut hashed).unwrap(); + Ok(State { + graph, + db, + hashes: hashed, + default: defaults, + pools: SmallMap::default(), + }) +} diff --git a/crates/moonbuild/src/lib.rs b/crates/moonbuild/src/lib.rs index 67b687ac..ca5c4816 100644 --- a/crates/moonbuild/src/lib.rs +++ b/crates/moonbuild/src/lib.rs @@ -30,6 +30,7 @@ pub mod entry; pub mod expect; pub mod fmt; pub mod gen; +pub mod generate; pub mod new; pub mod runtest; pub mod section_capture; diff --git a/crates/moonbuild/src/new.rs b/crates/moonbuild/src/new.rs index 2df092b2..bc324c79 100644 --- a/crates/moonbuild/src/new.rs +++ b/crates/moonbuild/src/new.rs @@ -71,6 +71,7 @@ pub fn moon_new_exec( warn_list: None, alert_list: None, targets: None, + pre_build: None, }; moonutil::common::write_package_json_to_file(&j, &main_moon_pkg)?; } @@ -213,6 +214,7 @@ fn common( warn_list: None, alert_list: None, targets: None, + pre_build: None, }; moonutil::common::write_package_json_to_file(&j, &lib_moon_pkg)?; } diff --git a/crates/moonbuild/template/pkg.schema.json b/crates/moonbuild/template/pkg.schema.json index b854372c..6ee6c002 100644 --- a/crates/moonbuild/template/pkg.schema.json +++ b/crates/moonbuild/template/pkg.schema.json @@ -45,6 +45,16 @@ "null" ] }, + "pre-build": { + "description": "Command for moon generate", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/MoonPkgGenerate" + } + }, "targets": { "description": "Conditional compilation targets", "type": [ @@ -166,6 +176,25 @@ } } }, + "MoonPkgGenerate": { + "type": "object", + "required": [ + "command", + "input", + "output" + ], + "properties": { + "command": { + "type": "string" + }, + "input": { + "$ref": "#/definitions/StringOrArray" + }, + "output": { + "$ref": "#/definitions/StringOrArray" + } + } + }, "PkgJSONImport": { "anyOf": [ { @@ -208,6 +237,19 @@ } ] }, + "StringOrArray": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, "WasmGcLinkConfig": { "type": "object", "properties": { diff --git a/crates/moonutil/src/package.rs b/crates/moonutil/src/package.rs index 0d824a68..cbf36ba5 100644 --- a/crates/moonutil/src/package.rs +++ b/crates/moonutil/src/package.rs @@ -59,6 +59,7 @@ pub struct Package { pub alert_list: Option, pub targets: Option>, + pub pre_build: Option>, } impl Package { @@ -190,6 +191,12 @@ pub struct MoonPkgJSON { #[schemars(rename = "targets")] #[schemars(with = "Option>>")] pub targets: Option, + + /// Command for moon generate + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(alias = "pre-build")] + #[schemars(rename = "pre-build")] + pub pre_build: Option>, } #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] @@ -283,6 +290,20 @@ pub struct Link { pub js: Option, } +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] +pub struct MoonPkgGenerate { + pub input: StringOrArray, + pub output: StringOrArray, + pub command: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] +#[serde(untagged)] +pub enum StringOrArray { + String(String), + Array(Vec), +} + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct MoonPkg { pub name: Option, @@ -297,6 +318,8 @@ pub struct MoonPkg { pub alert_list: Option, pub targets: Option, + + pub pre_build: Option>, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -458,6 +481,7 @@ pub fn convert_pkg_json_to_package(j: MoonPkgJSON) -> anyhow::Result { warn_list: j.warn_list, alert_list: j.alert_list, targets: j.targets, + pre_build: j.pre_build, }; Ok(result) } diff --git a/crates/moonutil/src/scan.rs b/crates/moonutil/src/scan.rs index c9f91260..d90cacdc 100644 --- a/crates/moonutil/src/scan.rs +++ b/crates/moonutil/src/scan.rs @@ -347,6 +347,7 @@ fn scan_one_package( warn_list, alert_list, targets: cond_targets, + pre_build: pkg.pre_build, }; if doc_mode { // -o diff --git a/docs/manual/src/source/pkg_json_schema.html b/docs/manual/src/source/pkg_json_schema.html index 02936ef4..b8bc7603 100644 --- a/docs/manual/src/source/pkg_json_schema.html +++ b/docs/manual/src/source/pkg_json_schema.html @@ -83,6 +83,16 @@ "null" ] }, + "pre-build": { + "description": "Command for moon generate", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/MoonPkgGenerate" + } + }, "targets": { "description": "Conditional compilation targets", "type": [ @@ -204,6 +214,25 @@ } } }, + "MoonPkgGenerate": { + "type": "object", + "required": [ + "command", + "input", + "output" + ], + "properties": { + "command": { + "type": "string" + }, + "input": { + "$ref": "#/definitions/StringOrArray" + }, + "output": { + "$ref": "#/definitions/StringOrArray" + } + } + }, "PkgJSONImport": { "anyOf": [ { @@ -246,6 +275,19 @@ } ] }, + "StringOrArray": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, "WasmGcLinkConfig": { "type": "object", "properties": {