split rust code into crates

so that we get more parallelism out of cargo

Author: folkertdev

crates/intrinsic-test/src/arm/config.rs

@@ -114,7 +114,6 @@ pub const AARCH_CONFIGURATIONS: &str = r#"
 #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_fcma))]
 #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_dotprod))]
 #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_i8mm))]
-#![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_sha3))]
 #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_sm4))]
 #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_ftts))]
 #![feature(fmt_helpers_for_derive)]

crates/intrinsic-test/src/arm/mod.rs

@@ -13,7 +13,9 @@ use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
 use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
-use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs};
+use crate::common::gen_rust::{
+    compile_rust_programs, write_bin_cargo_toml, write_lib_cargo_toml, write_main_rs,
+};
 use crate::common::intrinsic::Intrinsic;
 use crate::common::intrinsic_helpers::TypeKind;
 use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
@@ -137,19 +139,20 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
             "aarch64"
         };
 
+        // Experimentally, keeping 2 cores free is fastest for cargo.
         let available_parallelism = std::thread::available_parallelism().unwrap().get();
-        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
+        let chunk_count = Ord::min(available_parallelism, self.intrinsics.len());
+        let chunk_size = self.intrinsics.len().div_ceil(chunk_count);
 
         let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
-        write_cargo_toml(&mut cargo, &[]).unwrap();
+        write_bin_cargo_toml(&mut cargo, chunk_count).unwrap();
 
         let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
         write_main_rs(
             &mut main_rs,
-            available_parallelism,
-            architecture,
+            chunk_count,
             AARCH_CONFIGURATIONS,
-            F16_FORMATTING_DEF,
+            "",
             self.intrinsics.iter().map(|i| i.name.as_str()),
         )
         .unwrap();
@@ -165,19 +168,41 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
             .map(|(i, chunk)| {
                 use std::io::Write;
 
-                let rust_filename = format!("rust_programs/src/mod_{i}.rs");
+                std::fs::create_dir_all(format!("rust_programs/mod_{i}/src"))?;
+
+                let rust_filename = format!("rust_programs/mod_{i}/src/lib.rs");
                 trace!("generating `{rust_filename}`");
                 let mut file = File::create(rust_filename).unwrap();
 
                 write!(file, "{notice}")?;
 
+                writeln!(file, "#![feature(simd_ffi)]")?;
+                writeln!(file, "#![feature(f16)]")?;
+                writeln!(file, "#![allow(unused)]")?;
+
+                // Cargo will spam the logs if these warnings are not silenced.
+                writeln!(file, "#![allow(non_upper_case_globals)]")?;
+                writeln!(file, "#![allow(non_camel_case_types)]")?;
+                writeln!(file, "#![allow(non_snake_case)]")?;
+
+                let cfg = AARCH_CONFIGURATIONS;
+                writeln!(file, "{cfg}")?;
+
                 writeln!(file, "use core_arch::arch::{architecture}::*;")?;
-                writeln!(file, "use crate::{{debug_simd_finish, debug_f16}};")?;
+
+                let definitions = F16_FORMATTING_DEF;
+                writeln!(file, "{definitions}")?;
 
                 for intrinsic in chunk {
                     crate::common::gen_rust::create_rust_test_module(&mut file, intrinsic)?;
                 }
 
+                let toml_filename = format!("rust_programs/mod_{i}/Cargo.toml");
+                trace!("generating `{toml_filename}`");
+                let mut file = File::create(toml_filename).unwrap();
+
+                write_lib_cargo_toml(&mut file, &format!("mod_{i}"))?;
+
                 Ok(())
             })
             .collect::<Result<(), std::io::Error>>()

crates/intrinsic-test/src/common/compare.rs

@@ -24,6 +24,7 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
             let rust = runner_command(runner)
                 .arg(format!("target/{target}/release/intrinsic-test-programs"))
                 .arg(intrinsic_name)
+                .current_dir("rust_programs")
                 .output();
 
             let (c, rust) = match (c, rust) {

crates/intrinsic-test/src/common/gen_rust.rs

@@ -9,46 +9,53 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition;
 // The number of times each intrinsic will be called.
 const PASSES: u32 = 20;
 
-pub fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> {
+fn write_cargo_toml_header(w: &mut impl std::io::Write, name: &str) -> std::io::Result<()> {
     writeln!(
         w,
         concat!(
             "[package]\n",
-            "name = \"intrinsic-test-programs\"\n",
+            "name = \"{name}\"\n",
             "version = \"{version}\"\n",
             "authors = [{authors}]\n",
             "license = \"{license}\"\n",
             "edition = \"2018\"\n",
-            "[workspace]\n",
-            "[dependencies]\n",
-            "core_arch = {{ path = \"../crates/core_arch\" }}",
         ),
+        name = name,
         version = env!("CARGO_PKG_VERSION"),
         authors = env!("CARGO_PKG_AUTHORS")
             .split(":")
             .format_with(", ", |author, fmt| fmt(&format_args!("\"{author}\""))),
         license = env!("CARGO_PKG_LICENSE"),
-    )?;
+    )
+}
 
-    for binary in binaries {
-        writeln!(
-            w,
-            concat!(
-                "[[bin]]\n",
-                "name = \"{binary}\"\n",
-                "path = \"{binary}/main.rs\"\n",
-            ),
-            binary = binary,
-        )?;
+pub fn write_bin_cargo_toml(
+    w: &mut impl std::io::Write,
+    module_count: usize,
+) -> std::io::Result<()> {
+    write_cargo_toml_header(w, "intrinsic-test-programs")?;
+
+    writeln!(w, "[dependencies]")?;
+
+    for i in 0..module_count {
+        writeln!(w, "mod_{i} = {{ path = \"mod_{i}/\" }}")?;
     }
 
     Ok(())
 }
 
+pub fn write_lib_cargo_toml(w: &mut impl std::io::Write, name: &str) -> std::io::Result<()> {
+    write_cargo_toml_header(w, name)?;
+
+    writeln!(w, "[dependencies]")?;
+    writeln!(w, "core_arch = {{ path = \"../../crates/core_arch\" }}")?;
+
+    Ok(())
+}
+
 pub fn write_main_rs<'a>(
     w: &mut impl std::io::Write,
-    available_parallelism: usize,
-    architecture: &str,
+    chunk_count: usize,
     cfg: &str,
     definitions: &str,
     intrinsics: impl Iterator<Item = &'a str> + Clone,
@@ -65,10 +72,7 @@ pub fn write_main_rs<'a>(
     writeln!(w, "{cfg}")?;
     writeln!(w, "{definitions}")?;
 
-    writeln!(w, "use core_arch::arch::{architecture}::*;")?;
-
-    for module in 0..Ord::min(available_parallelism, intrinsics.clone().count()) {
-        writeln!(w, "mod mod_{module};")?;
+    for module in 0..chunk_count {
         writeln!(w, "use mod_{module}::*;")?;
     }
 
@@ -100,6 +104,9 @@ pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Opti
     let mut cargo_command = Command::new("cargo");
     cargo_command.current_dir("rust_programs");
 
+    // Do not use the target directory of the workspace please.
+    cargo_command.env("CARGO_TARGET_DIR", "target");
+
     if let Some(toolchain) = toolchain
         && !toolchain.is_empty()
     {