From 64536394e0bba3c33ba08b8e5300f35b01451b63 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Wed, 2 Oct 2024 22:49:15 -0400 Subject: [PATCH 1/9] First try at reverse tracing --- Cargo.lock | 3 + crates/turbo-trace/Cargo.toml | 3 + crates/turbo-trace/src/main.rs | 15 +- crates/turbo-trace/src/tracer.rs | 243 ++++++++++++++++--------- crates/turborepo-lib/src/query/file.rs | 10 +- 5 files changed, 190 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae5ea50a5fe37..622b7330552f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6066,6 +6066,7 @@ version = "0.1.0" dependencies = [ "camino", "clap", + "globwalk", "miette", "oxc_resolver 2.0.0", "swc_common", @@ -6073,6 +6074,8 @@ dependencies = [ "swc_ecma_parser", "swc_ecma_visit", "thiserror", + "tracing", + "tracing-subscriber", "turbopath", ] diff --git a/crates/turbo-trace/Cargo.toml b/crates/turbo-trace/Cargo.toml index 80526b52ca1b2..f909cc0262e51 100644 --- a/crates/turbo-trace/Cargo.toml +++ b/crates/turbo-trace/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] camino.workspace = true clap = { version = "4.5.17", features = ["derive"] } +globwalk = { version = "0.1.0", path = "../turborepo-globwalk" } miette = { workspace = true, features = ["fancy"] } oxc_resolver = { version = "2.0.0" } swc_common = { workspace = true } @@ -14,6 +15,8 @@ swc_ecma_ast = { workspace = true } swc_ecma_parser = { workspace = true } swc_ecma_visit = { workspace = true } thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } turbopath = { workspace = true } [lints] diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index f9b13a344a5c8..e29c5a224089e 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -3,6 +3,7 @@ mod tracer; use camino::Utf8PathBuf; use clap::Parser; +use miette::Report; use tracer::Tracer; use turbopath::{AbsoluteSystemPathBuf, PathError}; @@ -14,10 +15,16 @@ struct Args { ts_config: Option, files: Vec, #[clap(long)] +<<<<<<< HEAD depth: Option, +||||||| parent of d057b6922b (First try at reverse tracing) +======= + reverse: bool, +>>>>>>> d057b6922b (First try at reverse tracing) } fn main() -> Result<(), PathError> { + tracing_subscriber::fmt::init(); let args = Args::parse(); let abs_cwd = if let Some(cwd) = args.cwd { @@ -34,7 +41,11 @@ fn main() -> Result<(), PathError> { let tracer = Tracer::new(abs_cwd, files, args.ts_config); - let result = tracer.trace(args.depth); + let result = if args.reverse { + tracer.reverse_trace() + } else { + tracer.trace(args.depth) + }; if !result.errors.is_empty() { for error in &result.errors { @@ -42,7 +53,7 @@ fn main() -> Result<(), PathError> { } std::process::exit(1); } else { - for file in result.files.keys() { + for file in &result.files { println!("{}", file); } } diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index e1b0e2cbaa39d..864e882d613a2 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, fs, rc::Rc}; use camino::Utf8PathBuf; +use globwalk::WalkType; use miette::{Diagnostic, NamedSource, SourceSpan}; use oxc_resolver::{ EnforceExtension, ResolveError, ResolveOptions, Resolver, TsconfigOptions, TsconfigReferences, @@ -10,7 +11,7 @@ use swc_ecma_ast::EsVersion; use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::VisitWith; use thiserror::Error; -use turbopath::{AbsoluteSystemPathBuf, PathError}; +use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; @@ -23,23 +24,29 @@ pub struct Tracer { files: Vec<(AbsoluteSystemPathBuf, usize)>, ts_config: Option, source_map: Rc, + cwd: AbsoluteSystemPathBuf, } #[derive(Debug, Error, Diagnostic)] pub enum TraceError { + #[error("failed to parse file: {:?}", .0)] + ParseError(swc_ecma_parser::error::Error), #[error("failed to read file: {0}")] FileNotFound(AbsoluteSystemPathBuf), #[error(transparent)] PathEncoding(PathError), #[error("tracing a root file `{0}`, no parent found")] RootFile(AbsoluteSystemPathBuf), - #[error("failed to resolve import")] + #[error("failed to resolve import to `{path}`")] Resolve { + path: String, #[label("import here")] span: SourceSpan, #[source_code] text: NamedSource, }, + #[error("failed to walk files")] + GlobError(#[from] globwalk::WalkError), } pub struct TraceResult { @@ -61,11 +68,110 @@ impl Tracer { Self { files, ts_config, + cwd, source_map: Rc::new(SourceMap::default()), } } - pub fn trace(mut self, max_depth: Option) -> TraceResult { + pub fn get_imports_from_file( + &self, + resolver: &Resolver, + file_path: &AbsoluteSystemPath, + ) -> Result<(Vec, SeenFile), TraceError> { + // Read the file content + let Ok(file_content) = fs::read_to_string(&file_path) else { + return Err(TraceError::FileNotFound(file_path.to_owned())); + }; + + let comments = SingleThreadedComments::default(); + + let source_file = self.source_map.new_source_file( + FileName::Custom(file_path.to_string()).into(), + file_content.clone(), + ); + + let syntax = if file_path.extension() == Some("ts") || file_path.extension() == Some("tsx") + { + Syntax::Typescript(TsSyntax { + tsx: file_path.extension() == Some("tsx"), + decorators: true, + ..Default::default() + }) + } else { + Syntax::Es(EsSyntax { + jsx: file_path.extension() == Some("jsx"), + ..Default::default() + }) + }; + + let lexer = Lexer::new( + syntax, + EsVersion::EsNext, + StringInput::from(&*source_file), + Some(&comments), + ); + + let mut parser = Parser::new_from(Capturing::new(lexer)); + + // Parse the file as a module + let Ok(module) = parser.parse_module() else { + return Err(TraceError::FileNotFound(file_path.to_owned())); + }; + + // Visit the AST and find imports + let mut finder = ImportFinder::default(); + module.visit_with(&mut finder); + // Convert found imports/requires to absolute paths and add them to files to + // visit + let mut files = Vec::new(); + for (import, span) in finder.imports() { + let Some(file_dir) = file_path.parent() else { + return Err(TraceError::RootFile(file_path.to_owned())); + }; + match resolver.resolve(file_dir, import) { + Ok(resolved) => match resolved.into_path_buf().try_into() { + Ok(path) => files.push(path), + Err(err) => { + return Err(TraceError::PathEncoding(err)); + } + }, + Err(ResolveError::Builtin { .. }) => {} + Err(_) => { + let (start, end) = self.source_map.span_to_char_offset(&source_file, *span); + + return Err(TraceError::Resolve { + path: import.to_string(), + span: (start as usize, end as usize).into(), + text: NamedSource::new(file_path.to_string(), file_content.clone()), + }); + } + } + } + + Ok((files, SeenFile { ast: Some(module) })) + } + + pub fn trace_file( + &mut self, + resolver: &Resolver, + file_path: AbsoluteSystemPathBuf, + depth: usize, + seen: &mut HashMap, + ) -> Result<(), TraceError> { + if seen.contains_key(&file_path) { + return Ok(()); + } + + let (imports, seen_file) = self.get_imports_from_file(resolver, &file_path)?; + self.files + .extend(imports.into_iter().map(|import| (import, depth + 1))); + + seen.insert(file_path, seen_file); + + Ok(()) + } + + pub fn create_resolver(&mut self) -> Resolver { let mut options = ResolveOptions::default() .with_builtin_modules(true) .with_force_extension(EnforceExtension::Disabled) @@ -79,104 +185,79 @@ impl Tracer { }); } - let resolver = Resolver::new(options); + Resolver::new(options) + } + + pub fn trace(mut self, max_depth: Option) -> TraceResult { let mut errors = vec![]; let mut seen: HashMap = HashMap::new(); + let resolver = self.create_resolver(); while let Some((file_path, file_depth)) = self.files.pop() { - if matches!(file_path.extension(), Some("json") | Some("css")) { - continue; - } - - if seen.contains_key(&file_path) { - continue; - } - if let Some(max_depth) = max_depth { if file_depth > max_depth { continue; } } + if let Err(err) = self.trace_file(&resolver, file_path, file_depth, &mut seen) { + errors.push(err); + } + } - let entry = seen.entry(file_path.clone()).or_default(); - - // Read the file content - let Ok(file_content) = fs::read_to_string(&file_path) else { - errors.push(TraceError::FileNotFound(file_path.clone())); - continue; - }; - - let comments = SingleThreadedComments::default(); - - let source_file = self.source_map.new_source_file( - FileName::Custom(file_path.to_string()).into(), - file_content.clone(), - ); - - let syntax = - if file_path.extension() == Some("ts") || file_path.extension() == Some("tsx") { - Syntax::Typescript(TsSyntax { - tsx: file_path.extension() == Some("tsx"), - decorators: true, - ..Default::default() - }) - } else { - Syntax::Es(EsSyntax { - jsx: file_path.ends_with(".jsx"), - ..Default::default() - }) - }; - - let lexer = Lexer::new( - syntax, - EsVersion::EsNext, - StringInput::from(&*source_file), - Some(&comments), - ); - - let mut parser = Parser::new_from(Capturing::new(lexer)); - - // Parse the file as a module - let Ok(module) = parser.parse_module() else { - errors.push(TraceError::FileNotFound(file_path.to_owned())); - continue; - }; + TraceResult { + files: seen, + errors, + } + } - // Visit the AST and find imports - let mut finder = ImportFinder::default(); - module.visit_with(&mut finder); + pub fn reverse_trace(mut self) -> TraceResult { + let files = match globwalk::globwalk( + &self.cwd, + &[ + "**/*.ts".parse().expect("valid glob"), + "**/*.tsx".parse().expect("valid glob"), + ], + &[ + "**/node_modules/**".parse().expect("valid glob"), + "**/.next/**".parse().expect("valid glob"), + ], + WalkType::Files, + ) { + Ok(files) => files, + Err(e) => { + return TraceResult { + files: HashMap::new(), + errors: vec![e.into()], + } + } + }; - entry.ast = Some(module); + let resolver = self.create_resolver(); - // Convert found imports/requires to absolute paths and add them to files to - // visit - for (import, span) in finder.imports() { - let Some(file_dir) = file_path.parent() else { - errors.push(TraceError::RootFile(file_path.to_owned())); - continue; - }; - match resolver.resolve(file_dir, import) { - Ok(resolved) => match resolved.into_path_buf().try_into() { - Ok(path) => self.files.push((path, file_depth + 1)), - Err(err) => { - errors.push(TraceError::PathEncoding(err)); + let mut usages = HashMap::new(); + let mut errors = Vec::new(); + for file in files { + match self.get_imports_from_file(&resolver, &file) { + Ok((imported_files, seen_file)) => { + for import in imported_files { + if self + .files + .iter() + .any(|(source, _)| import.as_path() == source.as_path()) + { + usages.insert(file, seen_file); + break; } - }, - Err(ResolveError::Builtin { .. }) => {} - Err(_) => { - let (start, end) = self.source_map.span_to_char_offset(&source_file, *span); - - errors.push(TraceError::Resolve { - span: (start as usize, end as usize).into(), - text: NamedSource::new(file_path.to_string(), file_content.clone()), - }); } } + Err(e) => { + errors.push(e); + } } } TraceResult { - files: seen, + files: usages, errors, } } diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index 8c3e8ac14d2e6..a5f17c925e885 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -96,7 +96,15 @@ impl From for TraceError { path: Some(path.to_string()), ..Default::default() }, - turbo_trace::TraceError::Resolve { span, text } => { + turbo_trace::TraceError::ParseError(e) => TraceError { + message: format!("failed to parse file: {:?}", e), + ..Default::default() + }, + turbo_trace::TraceError::GlobError(_) => TraceError { + message: format!("failed to glob files"), + ..Default::default() + }, + turbo_trace::TraceError::Resolve { span, text, .. } => { let import = text .inner() .read_span(&span, 1, 1) From 0d10bab45649c3220621bed9c3d8ead9c71823dd Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Fri, 4 Oct 2024 15:00:35 -0400 Subject: [PATCH 2/9] Add config for node modules --- crates/turbo-trace/src/main.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index e29c5a224089e..aa8e41f568e93 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -13,14 +13,12 @@ struct Args { cwd: Option, #[clap(long)] ts_config: Option, + #[clap(long)] + node_modules: Option, files: Vec, #[clap(long)] -<<<<<<< HEAD depth: Option, -||||||| parent of d057b6922b (First try at reverse tracing) -======= reverse: bool, ->>>>>>> d057b6922b (First try at reverse tracing) } fn main() -> Result<(), PathError> { From ac899abbc34adc709c52b62621d8897764efdb7b Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Thu, 10 Oct 2024 15:02:12 -0400 Subject: [PATCH 3/9] Add to query --- crates/turborepo-lib/src/query/file.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index a5f17c925e885..c5db78197e925 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -185,6 +185,29 @@ impl File { TraceResult::new(result, self.run.clone()) } + async fn dependents(&self, ts_config: Option) -> TraceResult { + let ts_config = match ts_config { + Some(ts_config) => Some(Utf8PathBuf::from(ts_config)), + None => self + .path + .ancestors() + .skip(1) + .find(|p| p.join_component("tsconfig.json").exists()) + .map(|p| p.as_path().to_owned()), + }; + + let tracer = Tracer::new( + self.run.repo_root().to_owned(), + vec![self.path.clone()], + ts_config, + ); + + let mut result = tracer.reverse_trace(); + // Remove the file itself from the result + result.files.remove(&self.path); + TraceResult::new(result, self.run.clone()) + } + async fn ast(&self) -> Option { if let Some(ast) = &self.ast { serde_json::to_value(ast).ok() From b00aafa89f94f83903c24d20821ab1bf7641acde Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Tue, 15 Oct 2024 13:23:38 -0400 Subject: [PATCH 4/9] Attempting to debug some trace issues --- Cargo.lock | 6 +++--- crates/turbo-trace/src/tracer.rs | 1 + crates/turborepo-lib/src/query/file.rs | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 622b7330552f2..9bf7b35302d0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -718,7 +718,7 @@ dependencies = [ "biome_text_size", "bitflags 2.5.0", "bpaf", - "oxc_resolver 1.11.0", + "oxc_resolver 1.12.0", "serde", "termcolor", "unicode-width", @@ -3580,9 +3580,9 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "oxc_resolver" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fe4d07afdfcf6b1d7fb952e6691d82692a54b71964a377cf49f3e47dac283d" +checksum = "6c20bb345f290c46058ba650fef7ca2b579612cf2786b927ebad7b8bec0845a7" dependencies = [ "cfg-if", "dashmap 6.1.0", diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 864e882d613a2..5a071a9e5d1c7 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -185,6 +185,7 @@ impl Tracer { }); } + println!("Resolver options: {:?}", options); Resolver::new(options) } diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index c5db78197e925..c689b6b2b32f6 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -3,8 +3,10 @@ use std::sync::Arc; use async_graphql::{Object, SimpleObject}; use camino::Utf8PathBuf; use itertools::Itertools; +use miette::Report; use swc_ecma_ast::EsVersion; use swc_ecma_parser::{EsSyntax, Syntax, TsSyntax}; +use tracing::error; use turbo_trace::Tracer; use turbopath::AbsoluteSystemPathBuf; From f86b92f800b4a7b1430314193ef5f5fd44d69ae1 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Tue, 22 Oct 2024 18:06:05 -0400 Subject: [PATCH 5/9] Reverse turbo trace with some concurrency --- Cargo.lock | 3 ++ crates/turbo-trace/Cargo.toml | 4 +- crates/turbo-trace/src/main.rs | 7 +-- crates/turbo-trace/src/tracer.rs | 74 ++++++++++++++++---------- crates/turborepo-lib/src/query/file.rs | 6 +-- 5 files changed, 59 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bf7b35302d0b..f771fb4b19314 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5179,6 +5179,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", + "parking_lot", "rustc-hash 1.1.0", "serde", "siphasher", @@ -6066,6 +6067,7 @@ version = "0.1.0" dependencies = [ "camino", "clap", + "futures", "globwalk", "miette", "oxc_resolver 2.0.0", @@ -6074,6 +6076,7 @@ dependencies = [ "swc_ecma_parser", "swc_ecma_visit", "thiserror", + "tokio", "tracing", "tracing-subscriber", "turbopath", diff --git a/crates/turbo-trace/Cargo.toml b/crates/turbo-trace/Cargo.toml index f909cc0262e51..99f7fd50c3357 100644 --- a/crates/turbo-trace/Cargo.toml +++ b/crates/turbo-trace/Cargo.toml @@ -7,14 +7,16 @@ license = "MIT" [dependencies] camino.workspace = true clap = { version = "4.5.17", features = ["derive"] } +futures = { workspace = true } globwalk = { version = "0.1.0", path = "../turborepo-globwalk" } miette = { workspace = true, features = ["fancy"] } oxc_resolver = { version = "2.0.0" } -swc_common = { workspace = true } +swc_common = { workspace = true, features = ["concurrent"] } swc_ecma_ast = { workspace = true } swc_ecma_parser = { workspace = true } swc_ecma_visit = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } turbopath = { workspace = true } diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index aa8e41f568e93..721320ae69fc4 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -21,7 +21,8 @@ struct Args { reverse: bool, } -fn main() -> Result<(), PathError> { +#[tokio::main] +async fn main() -> Result<(), PathError> { tracing_subscriber::fmt::init(); let args = Args::parse(); @@ -40,9 +41,9 @@ fn main() -> Result<(), PathError> { let tracer = Tracer::new(abs_cwd, files, args.ts_config); let result = if args.reverse { - tracer.reverse_trace() + tracer.reverse_trace().await } else { - tracer.trace(args.depth) + tracer.trace(args.depth).await }; if !result.errors.is_empty() { diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 5a071a9e5d1c7..29dc3a22bc3f9 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fs, rc::Rc}; +use std::{collections::HashMap, sync::Arc}; use camino::Utf8PathBuf; use globwalk::WalkType; @@ -11,6 +11,7 @@ use swc_ecma_ast::EsVersion; use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::VisitWith; use thiserror::Error; +use tokio::{sync::Mutex, task::JoinSet}; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; @@ -23,7 +24,7 @@ pub struct SeenFile { pub struct Tracer { files: Vec<(AbsoluteSystemPathBuf, usize)>, ts_config: Option, - source_map: Rc, + source_map: Arc, cwd: AbsoluteSystemPathBuf, } @@ -69,17 +70,17 @@ impl Tracer { files, ts_config, cwd, - source_map: Rc::new(SourceMap::default()), + source_map: Arc::new(SourceMap::default()), } } - pub fn get_imports_from_file( + pub async fn get_imports_from_file( &self, resolver: &Resolver, file_path: &AbsoluteSystemPath, ) -> Result<(Vec, SeenFile), TraceError> { // Read the file content - let Ok(file_content) = fs::read_to_string(&file_path) else { + let Ok(file_content) = tokio::fs::read_to_string(&file_path).await else { return Err(TraceError::FileNotFound(file_path.to_owned())); }; @@ -151,18 +152,21 @@ impl Tracer { Ok((files, SeenFile { ast: Some(module) })) } - pub fn trace_file( + pub async fn trace_file( &mut self, resolver: &Resolver, file_path: AbsoluteSystemPathBuf, depth: usize, seen: &mut HashMap, ) -> Result<(), TraceError> { + if matches!(file_path.extension(), Some("css") | Some("json")) { + return Ok(()); + } if seen.contains_key(&file_path) { return Ok(()); } - let (imports, seen_file) = self.get_imports_from_file(resolver, &file_path)?; + let (imports, seen_file) = self.get_imports_from_file(resolver, &file_path).await?; self.files .extend(imports.into_iter().map(|import| (import, depth + 1))); @@ -185,11 +189,10 @@ impl Tracer { }); } - println!("Resolver options: {:?}", options); Resolver::new(options) } - pub fn trace(mut self, max_depth: Option) -> TraceResult { + pub async fn trace(mut self, max_depth: Option) -> TraceResult { let mut errors = vec![]; let mut seen: HashMap = HashMap::new(); let resolver = self.create_resolver(); @@ -200,7 +203,10 @@ impl Tracer { continue; } } - if let Err(err) = self.trace_file(&resolver, file_path, file_depth, &mut seen) { + if let Err(err) = self + .trace_file(&resolver, file_path, file_depth, &mut seen) + .await + { errors.push(err); } } @@ -211,7 +217,7 @@ impl Tracer { } } - pub fn reverse_trace(mut self) -> TraceResult { + pub async fn reverse_trace(mut self) -> TraceResult { let files = match globwalk::globwalk( &self.cwd, &[ @@ -233,27 +239,41 @@ impl Tracer { } }; - let resolver = self.create_resolver(); + let mut futures = JoinSet::new(); + + let resolver = Arc::new(self.create_resolver()); + let shared_self = Arc::new(self); - let mut usages = HashMap::new(); - let mut errors = Vec::new(); for file in files { - match self.get_imports_from_file(&resolver, &file) { - Ok((imported_files, seen_file)) => { - for import in imported_files { - if self - .files - .iter() - .any(|(source, _)| import.as_path() == source.as_path()) - { - usages.insert(file, seen_file); - break; - } + let shared_self = shared_self.clone(); + let resolver = resolver.clone(); + futures.spawn(async move { + let (imported_files, seen_file) = + shared_self.get_imports_from_file(&resolver, &file).await?; + for import in imported_files { + if shared_self + .files + .iter() + .any(|(source, _)| import.as_path() == source.as_path()) + { + return Ok(Some((file, seen_file))); } } - Err(e) => { - errors.push(e); + + Ok(None) + }); + } + + let mut usages = HashMap::new(); + let mut errors = Vec::new(); + + while let Some(result) = futures.join_next().await { + match result.unwrap() { + Ok(Some((path, file))) => { + usages.insert(path, file); } + Ok(None) => {} + Err(err) => errors.push(err), } } diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index c689b6b2b32f6..f3ff08d7d67f7 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -3,10 +3,8 @@ use std::sync::Arc; use async_graphql::{Object, SimpleObject}; use camino::Utf8PathBuf; use itertools::Itertools; -use miette::Report; use swc_ecma_ast::EsVersion; use swc_ecma_parser::{EsSyntax, Syntax, TsSyntax}; -use tracing::error; use turbo_trace::Tracer; use turbopath::AbsoluteSystemPathBuf; @@ -181,7 +179,7 @@ impl File { ts_config, ); - let mut result = tracer.trace(depth); + let mut result = tracer.trace(depth).await; // Remove the file itself from the result result.files.remove(&self.path); TraceResult::new(result, self.run.clone()) @@ -204,7 +202,7 @@ impl File { ts_config, ); - let mut result = tracer.reverse_trace(); + let mut result = tracer.reverse_trace().await; // Remove the file itself from the result result.files.remove(&self.path); TraceResult::new(result, self.run.clone()) From cef6e1aaf62d52a8dd3a0ef3da63d9790b5cc115 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Fri, 25 Oct 2024 13:10:06 -0400 Subject: [PATCH 6/9] Add more tests and fixing issues --- crates/turbo-trace/src/main.rs | 6 +- crates/turbo-trace/src/tracer.rs | 114 +++++++++++------- crates/turborepo/tests/query.rs | 35 +++++- ...ton.tsx`_with_dependents_(npm@10.5.0).snap | 23 ++++ ...id.ts`_with_dependencies_(npm@10.5.0).snap | 2 +- ...ix.ts`_with_dependencies_(npm@10.5.0).snap | 23 ++++ ...ex.ts`_with_dependencies_(npm@10.5.0).snap | 32 +++++ ...__index`_with_dependents_(npm@10.5.0).snap | 26 ++++ .../fixtures/turbo_trace/tsconfig.json | 9 ++ .../fixtures/turbo_trace/with_prefix.ts | 1 + .../fixtures/turbo_trace_monorepo/.gitignore | 3 + .../apps/my-app/.env.local | 0 .../turbo_trace_monorepo/apps/my-app/index.ts | 4 + .../apps/my-app/package.json | 10 ++ .../apps/my-app/tsconfig.json | 9 ++ .../turbo_trace_monorepo/apps/my-app/types.ts | 3 + .../turbo_trace_monorepo/package.json | 11 ++ .../packages/another/index.js | 3 + .../packages/another/package.json | 9 ++ .../packages/utils/index.ts | 2 + .../packages/utils/my-hook.ts | 3 + .../packages/utils/package.json | 12 ++ .../fixtures/turbo_trace_monorepo/turbo.json | 1 + 23 files changed, 293 insertions(+), 48 deletions(-) create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/.env.local create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index 721320ae69fc4..c1b97c6f86cdf 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -47,12 +47,12 @@ async fn main() -> Result<(), PathError> { }; if !result.errors.is_empty() { - for error in &result.errors { - eprintln!("error: {}", error); + for error in result.errors { + println!("{:?}", Report::new(error)) } std::process::exit(1); } else { - for file in &result.files { + for file in result.files.keys() { println!("{}", file); } } diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 29dc3a22bc3f9..27433093d5da8 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -11,7 +11,8 @@ use swc_ecma_ast::EsVersion; use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::VisitWith; use thiserror::Error; -use tokio::{sync::Mutex, task::JoinSet}; +use tokio::task::JoinSet; +use tracing::debug; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; @@ -26,6 +27,7 @@ pub struct Tracer { ts_config: Option, source_map: Arc, cwd: AbsoluteSystemPathBuf, + errors: Vec, } #[derive(Debug, Error, Diagnostic)] @@ -70,23 +72,26 @@ impl Tracer { files, ts_config, cwd, + errors: Vec::new(), source_map: Arc::new(SourceMap::default()), } } pub async fn get_imports_from_file( - &self, + source_map: &SourceMap, + errors: &mut Vec, resolver: &Resolver, file_path: &AbsoluteSystemPath, - ) -> Result<(Vec, SeenFile), TraceError> { + ) -> Option<(Vec, SeenFile)> { // Read the file content let Ok(file_content) = tokio::fs::read_to_string(&file_path).await else { - return Err(TraceError::FileNotFound(file_path.to_owned())); + errors.push(TraceError::FileNotFound(file_path.to_owned())); + return None; }; let comments = SingleThreadedComments::default(); - let source_file = self.source_map.new_source_file( + let source_file = source_map.new_source_file( FileName::Custom(file_path.to_string()).into(), file_content.clone(), ); @@ -116,7 +121,8 @@ impl Tracer { // Parse the file as a module let Ok(module) = parser.parse_module() else { - return Err(TraceError::FileNotFound(file_path.to_owned())); + errors.push(TraceError::FileNotFound(file_path.to_owned())); + return None; }; // Visit the AST and find imports @@ -126,30 +132,40 @@ impl Tracer { // visit let mut files = Vec::new(); for (import, span) in finder.imports() { + debug!("processing {} in {}", import, file_path); let Some(file_dir) = file_path.parent() else { - return Err(TraceError::RootFile(file_path.to_owned())); + errors.push(TraceError::RootFile(file_path.to_owned())); + continue; }; match resolver.resolve(file_dir, import) { - Ok(resolved) => match resolved.into_path_buf().try_into() { - Ok(path) => files.push(path), - Err(err) => { - return Err(TraceError::PathEncoding(err)); + Ok(resolved) => { + debug!("resolved {:?}", resolved); + match resolved.into_path_buf().try_into() { + Ok(path) => files.push(path), + Err(err) => { + errors.push(TraceError::PathEncoding(err)); + continue; + } } - }, - Err(ResolveError::Builtin { .. }) => {} - Err(_) => { - let (start, end) = self.source_map.span_to_char_offset(&source_file, *span); + } + Err(err @ ResolveError::Builtin { .. }) => { + debug!("built in: {:?}", err); + } + Err(err) => { + debug!("failed to resolve: {:?}", err); + let (start, end) = source_map.span_to_char_offset(&source_file, *span); - return Err(TraceError::Resolve { + errors.push(TraceError::Resolve { path: import.to_string(), span: (start as usize, end as usize).into(), text: NamedSource::new(file_path.to_string(), file_content.clone()), }); + continue; } } } - Ok((files, SeenFile { ast: Some(module) })) + Some((files, SeenFile { ast: Some(module) })) } pub async fn trace_file( @@ -158,21 +174,27 @@ impl Tracer { file_path: AbsoluteSystemPathBuf, depth: usize, seen: &mut HashMap, - ) -> Result<(), TraceError> { + ) { if matches!(file_path.extension(), Some("css") | Some("json")) { - return Ok(()); + return; } if seen.contains_key(&file_path) { - return Ok(()); + return; } - let (imports, seen_file) = self.get_imports_from_file(resolver, &file_path).await?; - self.files - .extend(imports.into_iter().map(|import| (import, depth + 1))); + let entry = seen.entry(file_path.clone()).or_default(); - seen.insert(file_path, seen_file); + let Some((imports, seen_file)) = + Self::get_imports_from_file(&self.source_map, &mut self.errors, resolver, &file_path) + .await + else { + return; + }; - Ok(()) + *entry = seen_file; + + self.files + .extend(imports.into_iter().map(|import| (import, depth + 1))); } pub fn create_resolver(&mut self) -> Resolver { @@ -193,7 +215,6 @@ impl Tracer { } pub async fn trace(mut self, max_depth: Option) -> TraceResult { - let mut errors = vec![]; let mut seen: HashMap = HashMap::new(); let resolver = self.create_resolver(); @@ -203,17 +224,13 @@ impl Tracer { continue; } } - if let Err(err) = self - .trace_file(&resolver, file_path, file_depth, &mut seen) - .await - { - errors.push(err); - } + self.trace_file(&resolver, file_path, file_depth, &mut seen) + .await; } TraceResult { files: seen, - errors, + errors: self.errors, } } @@ -221,6 +238,8 @@ impl Tracer { let files = match globwalk::globwalk( &self.cwd, &[ + "**/*.js".parse().expect("valid glob"), + "**/*.jsx".parse().expect("valid glob"), "**/*.ts".parse().expect("valid glob"), "**/*.tsx".parse().expect("valid glob"), ], @@ -248,19 +267,29 @@ impl Tracer { let shared_self = shared_self.clone(); let resolver = resolver.clone(); futures.spawn(async move { - let (imported_files, seen_file) = - shared_self.get_imports_from_file(&resolver, &file).await?; + let mut errors = Vec::new(); + let Some((imported_files, seen_file)) = Self::get_imports_from_file( + &shared_self.source_map, + &mut errors, + &resolver, + &file, + ) + .await + else { + return (errors, None); + }; + for import in imported_files { if shared_self .files .iter() .any(|(source, _)| import.as_path() == source.as_path()) { - return Ok(Some((file, seen_file))); + return (errors, Some((file, seen_file))); } } - Ok(None) + (errors, None) }); } @@ -268,12 +297,11 @@ impl Tracer { let mut errors = Vec::new(); while let Some(result) = futures.join_next().await { - match result.unwrap() { - Ok(Some((path, file))) => { - usages.insert(path, file); - } - Ok(None) => {} - Err(err) => errors.push(err), + let (errs, file) = result.unwrap(); + errors.extend(errs); + + if let Some((path, seen_file)) = file { + usages.insert(path, seen_file); } } diff --git a/crates/turborepo/tests/query.rs b/crates/turborepo/tests/query.rs index b6767a60c5b2c..5bc3d773846fe 100644 --- a/crates/turborepo/tests/query.rs +++ b/crates/turborepo/tests/query.rs @@ -21,7 +21,7 @@ fn test_double_symlink() -> Result<(), anyhow::Error> { } #[test] -fn test_trace() -> Result<(), anyhow::Error> { +fn test_ast() -> Result<(), anyhow::Error> { // Separate because the `\\` -> `/` filter isn't compatible with ast check_json!( "turbo_trace", @@ -30,6 +30,11 @@ fn test_trace() -> Result<(), anyhow::Error> { "get `main.ts` with ast" => "query { file(path: \"main.ts\") { path ast } }", ); + Ok(()) +} + +#[test] +fn test_trace() -> Result<(), anyhow::Error> { insta::with_settings!({ filters => vec![(r"\\\\", "/")]}, { check_json!( "turbo_trace", @@ -41,8 +46,36 @@ fn test_trace() -> Result<(), anyhow::Error> { "get `circular.ts` with dependencies" => "query { file(path: \"circular.ts\") { path dependencies { files { items { path } } } } }", "get `invalid.ts` with dependencies" => "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", "get `main.ts` with depth = 0" => "query { file(path: \"main.ts\") { path dependencies(depth: 1) { files { items { path } } } } }", + "get `with_prefix.ts` with dependencies" => "query { file(path: \"with_prefix.ts\") { path dependencies { files { items { path } } } } }", ); Ok(()) }) } + +#[test] +fn test_trace_on_monorepo() -> Result<(), anyhow::Error> { + insta::with_settings!({ filters => vec![(r"\\\\", "/")]}, { + check_json!( + "turbo_trace_monorepo", + "npm@10.5.0", + "query", + "get `apps/my-app/index.ts` with dependencies" => "query { file(path: \"apps/my-app/index.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", + "get `packages/utils/index` with dependents" => "query { file(path: \"packages/utils/index.ts\") { path dependents { files { items { path } } errors { items { message } } } } }", + ); + + Ok(()) + }) +} + +#[test] +fn test_reverse_trace() -> Result<(), anyhow::Error> { + check_json!( + "turbo_trace", + "npm@10.5.0", + "query", + "get `button.tsx` with dependents" => "query { file(path: \"button.tsx\") { path dependents { files { items { path } } } } }", + ); + + Ok(()) +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..e49db2b924ed6 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,23 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "button.tsx", + "dependents": { + "files": { + "items": [ + { + "path": "invalid.ts" + }, + { + "path": "main.ts" + } + ] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap index 8354df28773f7..55436189c5f86 100644 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap @@ -17,7 +17,7 @@ expression: query_output "errors": { "items": [ { - "message": "failed to resolve import" + "message": "failed to resolve import to `./non-existent-file.js`" } ] } diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap new file mode 100644 index 0000000000000..062887b12ac53 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap @@ -0,0 +1,23 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "with_prefix.ts", + "dependencies": { + "files": { + "items": [ + { + "path": "bar.js" + }, + { + "path": "foo.js" + } + ] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap new file mode 100644 index 0000000000000..62bfdc21cbe16 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap @@ -0,0 +1,32 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "apps/my-app/index.ts", + "dependencies": { + "files": { + "items": [ + { + "path": "apps/my-app/types.ts" + }, + { + "path": "packages/another/index.js" + }, + { + "path": "packages/utils/index.ts" + }, + { + "path": "packages/utils/my-hook.ts" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..4e6e7ed0e3329 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,26 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "packages/utils/index.ts", + "dependents": { + "files": { + "items": [ + { + "path": "apps/my-app/index.ts" + }, + { + "path": "packages/another/index.js" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json b/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json new file mode 100644 index 0000000000000..51d884f8a82d0 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "paths": { + "@*": [ + ".*" + ] + } + } +} \ No newline at end of file diff --git a/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts b/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts new file mode 100644 index 0000000000000..b5668f3729691 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts @@ -0,0 +1 @@ +import foo from "@/foo"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore new file mode 100644 index 0000000000000..77af9fc60321d --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.turbo +.npmrc diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/.env.local b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/.env.local new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts new file mode 100644 index 0000000000000..2b6c3821b3f30 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts @@ -0,0 +1,4 @@ +import { useMyHook } from "utils/my-hook"; +import ship from "utils"; +import { blackbeard } from "../../packages/another/index.js"; +import { Pirate } from "@/types.ts"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json new file mode 100644 index 0000000000000..cf17ebf161c4a --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json @@ -0,0 +1,10 @@ +{ + "name": "my-app", + "scripts": { + "build": "echo building", + "maybefails": "exit 4" + }, + "dependencies": { + "utils": "*" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json new file mode 100644 index 0000000000000..c199498e2ce9f --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "paths": { + "@/*": [ + "./*" + ] + } + } +} \ No newline at end of file diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts new file mode 100644 index 0000000000000..236ea60c41f5d --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts @@ -0,0 +1,3 @@ +export interface Pirate { + ship: string; +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json new file mode 100644 index 0000000000000..83e4fa43c6849 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json @@ -0,0 +1,11 @@ +{ + "name": "monorepo", + "scripts": { + "something": "turbo run build" + }, + "packageManager": "npm@10.5.0", + "workspaces": [ + "apps/**", + "packages/**" + ] +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js new file mode 100644 index 0000000000000..94639c6347160 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js @@ -0,0 +1,3 @@ +import ship from "utils"; + +export const blackbeard = "Edward Teach on " + ship; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json new file mode 100644 index 0000000000000..bb796c8455b16 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json @@ -0,0 +1,9 @@ +{ + "name": "another", + "scripts": { + "dev": "echo building" + }, + "dependencies": { + "utils": "*" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts new file mode 100644 index 0000000000000..5475301176b3b --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts @@ -0,0 +1,2 @@ +const ship = "The Queen Anne's Revenge"; +export default ship; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts new file mode 100644 index 0000000000000..dad43626ffbc4 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts @@ -0,0 +1,3 @@ +export const useMyHook = () => { + console.log("arrrr matey"); +}; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json new file mode 100644 index 0000000000000..2184abb4f4989 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json @@ -0,0 +1,12 @@ +{ + "name": "utils", + "scripts": { + "build": "echo building", + "maybefails": "echo didnotfail" + }, + "main": "index.ts", + "exports": { + ".": "./index.ts", + "./my-hook": "./my-hook.ts" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json new file mode 100644 index 0000000000000..9e26dfeeb6e64 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json @@ -0,0 +1 @@ +{} \ No newline at end of file From d0ab412891196480238c8157064e02e19d5f70f2 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Fri, 25 Oct 2024 14:23:03 -0400 Subject: [PATCH 7/9] Added tsconfig inference on each file. This allows us to resolve prefixes in a repo where we may have multiple tsconfigs --- crates/turbo-trace/src/tracer.rs | 36 ++++++++++++++++--- crates/turborepo-lib/src/query/file.rs | 24 ++----------- crates/turborepo/tests/query.rs | 3 +- ...ndex.js`_with_dependents_(npm@10.5.0).snap | 26 ++++++++++++++ ...ndex.ts`_with_dependents_(npm@10.5.0).snap | 26 ++++++++++++++ .../turbo_trace_monorepo/apps/my-app/types.ts | 2 ++ 6 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__another__index.js`_with_dependents_(npm@10.5.0).snap create mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index.ts`_with_dependents_(npm@10.5.0).snap diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 27433093d5da8..b61d4f556271e 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -175,6 +175,8 @@ impl Tracer { depth: usize, seen: &mut HashMap, ) { + let file_resolver = Self::infer_resolver_with_ts_config(&file_path, resolver); + if matches!(file_path.extension(), Some("css") | Some("json")) { return; } @@ -184,9 +186,13 @@ impl Tracer { let entry = seen.entry(file_path.clone()).or_default(); - let Some((imports, seen_file)) = - Self::get_imports_from_file(&self.source_map, &mut self.errors, resolver, &file_path) - .await + let Some((imports, seen_file)) = Self::get_imports_from_file( + &self.source_map, + &mut self.errors, + file_resolver.as_ref().unwrap_or(resolver), + &file_path, + ) + .await else { return; }; @@ -197,6 +203,26 @@ impl Tracer { .extend(imports.into_iter().map(|import| (import, depth + 1))); } + /// Attempts to find the closest tsconfig and creates a resolver with it, + /// so alias resolution, e.g. `@/foo/bar`, works. + fn infer_resolver_with_ts_config( + root: &AbsoluteSystemPath, + existing_resolver: &Resolver, + ) -> Option { + root.ancestors() + .skip(1) + .find(|p| p.join_component("tsconfig.json").exists()) + .map(|ts_config| { + let mut options = existing_resolver.options().clone(); + options.tsconfig = Some(TsconfigOptions { + config_file: ts_config.as_std_path().into(), + references: TsconfigReferences::Auto, + }); + + existing_resolver.clone_with_options(options) + }) + } + pub fn create_resolver(&mut self) -> Resolver { let mut options = ResolveOptions::default() .with_builtin_modules(true) @@ -267,11 +293,13 @@ impl Tracer { let shared_self = shared_self.clone(); let resolver = resolver.clone(); futures.spawn(async move { + let file_resolver = Self::infer_resolver_with_ts_config(&file, &resolver); let mut errors = Vec::new(); + let Some((imported_files, seen_file)) = Self::get_imports_from_file( &shared_self.source_map, &mut errors, - &resolver, + file_resolver.as_ref().unwrap_or(&resolver), &file, ) .await diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index f3ff08d7d67f7..7012328d41ef2 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -163,20 +163,10 @@ impl File { } async fn dependencies(&self, depth: Option, ts_config: Option) -> TraceResult { - let ts_config = match ts_config { - Some(ts_config) => Some(Utf8PathBuf::from(ts_config)), - None => self - .path - .ancestors() - .skip(1) - .find(|p| p.join_component("tsconfig.json").exists()) - .map(|p| p.as_path().to_owned()), - }; - let tracer = Tracer::new( self.run.repo_root().to_owned(), vec![self.path.clone()], - ts_config, + ts_config.map(Utf8PathBuf::from), ); let mut result = tracer.trace(depth).await; @@ -186,20 +176,10 @@ impl File { } async fn dependents(&self, ts_config: Option) -> TraceResult { - let ts_config = match ts_config { - Some(ts_config) => Some(Utf8PathBuf::from(ts_config)), - None => self - .path - .ancestors() - .skip(1) - .find(|p| p.join_component("tsconfig.json").exists()) - .map(|p| p.as_path().to_owned()), - }; - let tracer = Tracer::new( self.run.repo_root().to_owned(), vec![self.path.clone()], - ts_config, + ts_config.map(Utf8PathBuf::from), ); let mut result = tracer.reverse_trace().await; diff --git a/crates/turborepo/tests/query.rs b/crates/turborepo/tests/query.rs index 5bc3d773846fe..18308f173a9f5 100644 --- a/crates/turborepo/tests/query.rs +++ b/crates/turborepo/tests/query.rs @@ -61,7 +61,8 @@ fn test_trace_on_monorepo() -> Result<(), anyhow::Error> { "npm@10.5.0", "query", "get `apps/my-app/index.ts` with dependencies" => "query { file(path: \"apps/my-app/index.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", - "get `packages/utils/index` with dependents" => "query { file(path: \"packages/utils/index.ts\") { path dependents { files { items { path } } errors { items { message } } } } }", + "get `packages/utils/index.ts` with dependents" => "query { file(path: \"packages/utils/index.ts\") { path dependents { files { items { path } } errors { items { message } } } } }", + "get `packages/another/index.js` with dependents" => "query { file(path: \"packages/another/index.js\") { path dependents { files { items { path } } errors { items { message } } } } }", ); Ok(()) diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__another__index.js`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__another__index.js`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..d5fb69de4071c --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__another__index.js`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,26 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "packages/another/index.js", + "dependents": { + "files": { + "items": [ + { + "path": "apps/my-app/index.ts" + }, + { + "path": "apps/my-app/types.ts" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index.ts`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index.ts`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..4e6e7ed0e3329 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index.ts`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,26 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "packages/utils/index.ts", + "dependents": { + "files": { + "items": [ + { + "path": "apps/my-app/index.ts" + }, + { + "path": "packages/another/index.js" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts index 236ea60c41f5d..b20bb0af42f65 100644 --- a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts @@ -1,3 +1,5 @@ +import { blackbeard } from "@/../../packages/another/index.js"; + export interface Pirate { ship: string; } From a2fb6a5a84e22bc4e00cd2fdd1b3fc307fe3d64c Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Fri, 25 Oct 2024 14:29:44 -0400 Subject: [PATCH 8/9] Clippy --- crates/turborepo-lib/src/query/file.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index 7012328d41ef2..ce21775108c38 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -100,8 +100,8 @@ impl From for TraceError { message: format!("failed to parse file: {:?}", e), ..Default::default() }, - turbo_trace::TraceError::GlobError(_) => TraceError { - message: format!("failed to glob files"), + turbo_trace::TraceError::GlobError(err) => TraceError { + message: format!("failed to glob files: {}", err), ..Default::default() }, turbo_trace::TraceError::Resolve { span, text, .. } => { From 508ee07c0edf5ceba20cc3f9d8a54b1bf71bdbbc Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Fri, 25 Oct 2024 16:03:02 -0400 Subject: [PATCH 9/9] Fix test --- turborepo-tests/integration/tests/turbo-trace.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turborepo-tests/integration/tests/turbo-trace.t b/turborepo-tests/integration/tests/turbo-trace.t index 1cced6353923b..5896b4b6b3d2c 100644 --- a/turborepo-tests/integration/tests/turbo-trace.t +++ b/turborepo-tests/integration/tests/turbo-trace.t @@ -91,7 +91,7 @@ Trace file with invalid import "errors": { "items": [ { - "message": "failed to resolve import" + "message": "failed to resolve import to `./non-existent-file.js`" } ] }