add next-build-test binary / library

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "packages/next-swc/crates/napi",
   "packages/next-swc/crates/wasm",
   "packages/next-swc/crates/next-api",
+  "packages/next-swc/crates/next-build-test",
   "packages/next-swc/crates/next-build",
   "packages/next-swc/crates/next-core",
   "packages/next-swc/crates/next-custom-transforms",
@@ -22,6 +23,11 @@ debug-assertions = false
 [profile.dev.build-override]
 opt-level = 3
 
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+lto = "thin"
+
 [workspace.dependencies]
 # Workspace crates
 next-api = { path = "packages/next-swc/crates/next-api" }
@@ -42,6 +48,7 @@ turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbop
 turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240409.3" }
 # [TODO]: need to refactor embed_directory! macro usage in next-core
 turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240409.3" }
+turbo-tasks-malloc = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240409.3" }
 
 # General Deps
 

packages/next-swc/crates/next-build-test/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "next-build-test"
+version = "0.1.0"
+description = "TBD"
+license = "MPL-2.0"
+edition = "2021"
+autobenches = false
+
+[lints]
+workspace = true
+
+[dependencies]
+next-core = { workspace = true }
+next-api = { workspace = true }
+serde_json = { workspace = true }
+anyhow = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+turbo-tasks-malloc = { workspace = true, default-features = false }
+turbopack-binding = { workspace = true, features = [
+  "__turbo_tasks",
+  "__turbo_tasks_memory",
+  "__turbo_tasks_env",
+  "__turbo_tasks_fs",
+  "__turbo_tasks_memory",
+  "__turbopack",
+  "__turbopack_nodejs",
+  "__turbopack_core",
+  "__turbopack_browser",
+  "__turbopack_ecmascript",
+  "__turbopack_ecmascript_runtime",
+  "__turbopack_env",
+  "__turbopack_node",
+] }
+turbo-tasks = { workspace = true }
+rand.workspace = true
+num_cpus = "1.16.0"
+tokio-stream = "0.1.15"
+futures-util = "0.3.30"
+
+[build-dependencies]
+turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
+
+[dev-dependencies]
+criterion = { version = "0.5", features = ["async_tokio"] }

packages/next-swc/crates/next-build-test/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    turbopack_binding::turbo::tasks_build::generate_register();
+}

packages/next-swc/crates/next-build-test/src/lib.rs

@@ -0,0 +1,140 @@
+#![feature(future_join)]
+#![feature(min_specialization)]
+#![feature(arbitrary_self_types)]
+
+use std::str::FromStr;
+
+use anyhow::{Context, Result};
+use futures_util::{StreamExt, TryStreamExt};
+use next_api::{
+    project::{ProjectContainer, ProjectOptions},
+    route::{Endpoint, Route},
+};
+
+pub async fn main_inner(strat: Strategy, factor: usize, limit: usize) -> Result<()> {
+    register();
+
+    let mut file = std::fs::File::open("project_options.json").with_context(|| {
+        let path = std::env::current_dir()
+            .unwrap()
+            .join("project_options.json");
+        format!("loading file at {}", path.display())
+    })?;
+    let data: ProjectOptions = serde_json::from_reader(&mut file).unwrap();
+
+    let options = ProjectOptions { ..data };
+
+    let project = ProjectContainer::new(options);
+
+    println!("collecting endpoints");
+    let entrypoints = project.entrypoints().await?;
+
+    let routes = shuffle(entrypoints.routes.clone().into_iter());
+    render_routes(routes, strat, factor, limit).await;
+
+    Ok(())
+}
+
+pub fn register() {
+    next_api::register();
+    include!(concat!(env!("OUT_DIR"), "/register.rs"));
+}
+
+#[derive(PartialEq, Copy, Clone)]
+pub enum Strategy {
+    Sequential,
+    Concurrent,
+    Parallel,
+}
+
+impl std::fmt::Display for Strategy {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Strategy::Sequential => write!(f, "sequential"),
+            Strategy::Concurrent => write!(f, "concurrent"),
+            Strategy::Parallel => write!(f, "parallel"),
+        }
+    }
+}
+
+impl FromStr for Strategy {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        match s {
+            "sequential" => Ok(Strategy::Sequential),
+            "concurrent" => Ok(Strategy::Concurrent),
+            "parallel" => Ok(Strategy::Parallel),
+            _ => Err(anyhow::anyhow!("invalid strategy")),
+        }
+    }
+}
+
+pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
+    use rand::{seq::SliceRandom, thread_rng};
+    let mut rng = thread_rng();
+    let mut input = items.collect::<Vec<_>>();
+    input.shuffle(&mut rng);
+    input.into_iter()
+}
+
+pub async fn render_routes(
+    routes: impl Iterator<Item = (String, Route)>,
+    strategy: Strategy,
+    factor: usize,
+    limit: usize,
+) {
+    println!(
+        "rendering routes with {} parallel and strat {}",
+        factor, strategy
+    );
+
+    let stream = tokio_stream::iter(routes)
+        .map(move |(name, route)| {
+            let fut = async move {
+                println!("{name}");
+
+                match route {
+                    Route::Page {
+                        html_endpoint,
+                        data_endpoint: _,
+                    } => {
+                        html_endpoint.write_to_disk().await?;
+                    }
+                    Route::PageApi { endpoint } => {
+                        endpoint.write_to_disk().await?;
+                    }
+                    Route::AppPage(routes) => {
+                        for route in routes {
+                            route.html_endpoint.write_to_disk().await?;
+                        }
+                    }
+                    Route::AppRoute {
+                        original_name: _,
+                        endpoint,
+                    } => {
+                        endpoint.write_to_disk().await?;
+                    }
+                    Route::Conflict => {
+                        println!("WARN: conflict {}", name);
+                    }
+                }
+
+                Ok::<_, anyhow::Error>(())
+            };
+
+            async move {
+                match strategy {
+                    Strategy::Parallel => tokio::task::spawn(fut).await.unwrap(),
+                    _ => fut.await,
+                }
+            }
+        })
+        .take(limit)
+        .buffer_unordered(factor)
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+
+    println!("rendered {} pages", stream.len());
+}

packages/next-swc/crates/next-build-test/src/main.rs

@@ -0,0 +1,48 @@
+use std::str::FromStr;
+
+use next_build_test::{main_inner, Strategy};
+use turbo_tasks::TurboTasks;
+use turbo_tasks_malloc::TurboMalloc;
+use turbopack_binding::turbo::tasks_memory::MemoryBackend;
+
+// #[global_allocator]
+// static ALLOC: turbo_tasks_malloc::TurboMalloc =
+// turbo_tasks_malloc::TurboMalloc;
+
+fn main() {
+    let mut factor = std::env::args()
+        .nth(2)
+        .map(|s| s.parse().unwrap())
+        .unwrap_or(num_cpus::get());
+    let strat = std::env::args()
+        .nth(1)
+        .map(|s| Strategy::from_str(&s))
+        .transpose()
+        .unwrap()
+        .unwrap_or(Strategy::Sequential);
+
+    let limit = std::env::args()
+        .nth(3)
+        .map(|s| s.parse().unwrap())
+        .unwrap_or(1);
+
+    if strat == Strategy::Sequential {
+        factor = 1;
+    }
+
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .on_thread_stop(|| {
+            TurboMalloc::thread_stop();
+            println!("threads stopped");
+        })
+        .build()
+        .unwrap()
+        .block_on(async {
+            let tt = TurboTasks::new(MemoryBackend::new(usize::MAX));
+            let x = tt.run_once(main_inner(strat, factor, limit)).await;
+            println!("done");
+            x
+        })
+        .unwrap();
+}