diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs index 721320ae69fc4..c1b97c6f86cdf 100644 --- a/crates/turbo-trace/src/main.rs +++ b/crates/turbo-trace/src/main.rs @@ -47,12 +47,12 @@ async fn main() -> Result<(), PathError> { }; if !result.errors.is_empty() { - for error in &result.errors { - eprintln!("error: {}", error); + for error in result.errors { + println!("{:?}", Report::new(error)) } std::process::exit(1); } else { - for file in &result.files { + for file in result.files.keys() { println!("{}", file); } } diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 29dc3a22bc3f9..27433093d5da8 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -11,7 +11,8 @@ use swc_ecma_ast::EsVersion; use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::VisitWith; use thiserror::Error; -use tokio::{sync::Mutex, task::JoinSet}; +use tokio::task::JoinSet; +use tracing::debug; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; @@ -26,6 +27,7 @@ pub struct Tracer { ts_config: Option, source_map: Arc, cwd: AbsoluteSystemPathBuf, + errors: Vec, } #[derive(Debug, Error, Diagnostic)] @@ -70,23 +72,26 @@ impl Tracer { files, ts_config, cwd, + errors: Vec::new(), source_map: Arc::new(SourceMap::default()), } } pub async fn get_imports_from_file( - &self, + source_map: &SourceMap, + errors: &mut Vec, resolver: &Resolver, file_path: &AbsoluteSystemPath, - ) -> Result<(Vec, SeenFile), TraceError> { + ) -> Option<(Vec, SeenFile)> { // Read the file content let Ok(file_content) = tokio::fs::read_to_string(&file_path).await else { - return Err(TraceError::FileNotFound(file_path.to_owned())); + errors.push(TraceError::FileNotFound(file_path.to_owned())); + return None; }; let comments = SingleThreadedComments::default(); - let source_file = self.source_map.new_source_file( + let source_file = source_map.new_source_file( FileName::Custom(file_path.to_string()).into(), file_content.clone(), ); @@ -116,7 +121,8 @@ impl Tracer { // Parse the file as a module let Ok(module) = parser.parse_module() else { - return Err(TraceError::FileNotFound(file_path.to_owned())); + errors.push(TraceError::FileNotFound(file_path.to_owned())); + return None; }; // Visit the AST and find imports @@ -126,30 +132,40 @@ impl Tracer { // visit let mut files = Vec::new(); for (import, span) in finder.imports() { + debug!("processing {} in {}", import, file_path); let Some(file_dir) = file_path.parent() else { - return Err(TraceError::RootFile(file_path.to_owned())); + errors.push(TraceError::RootFile(file_path.to_owned())); + continue; }; match resolver.resolve(file_dir, import) { - Ok(resolved) => match resolved.into_path_buf().try_into() { - Ok(path) => files.push(path), - Err(err) => { - return Err(TraceError::PathEncoding(err)); + Ok(resolved) => { + debug!("resolved {:?}", resolved); + match resolved.into_path_buf().try_into() { + Ok(path) => files.push(path), + Err(err) => { + errors.push(TraceError::PathEncoding(err)); + continue; + } } - }, - Err(ResolveError::Builtin { .. }) => {} - Err(_) => { - let (start, end) = self.source_map.span_to_char_offset(&source_file, *span); + } + Err(err @ ResolveError::Builtin { .. }) => { + debug!("built in: {:?}", err); + } + Err(err) => { + debug!("failed to resolve: {:?}", err); + let (start, end) = source_map.span_to_char_offset(&source_file, *span); - return Err(TraceError::Resolve { + errors.push(TraceError::Resolve { path: import.to_string(), span: (start as usize, end as usize).into(), text: NamedSource::new(file_path.to_string(), file_content.clone()), }); + continue; } } } - Ok((files, SeenFile { ast: Some(module) })) + Some((files, SeenFile { ast: Some(module) })) } pub async fn trace_file( @@ -158,21 +174,27 @@ impl Tracer { file_path: AbsoluteSystemPathBuf, depth: usize, seen: &mut HashMap, - ) -> Result<(), TraceError> { + ) { if matches!(file_path.extension(), Some("css") | Some("json")) { - return Ok(()); + return; } if seen.contains_key(&file_path) { - return Ok(()); + return; } - let (imports, seen_file) = self.get_imports_from_file(resolver, &file_path).await?; - self.files - .extend(imports.into_iter().map(|import| (import, depth + 1))); + let entry = seen.entry(file_path.clone()).or_default(); - seen.insert(file_path, seen_file); + let Some((imports, seen_file)) = + Self::get_imports_from_file(&self.source_map, &mut self.errors, resolver, &file_path) + .await + else { + return; + }; - Ok(()) + *entry = seen_file; + + self.files + .extend(imports.into_iter().map(|import| (import, depth + 1))); } pub fn create_resolver(&mut self) -> Resolver { @@ -193,7 +215,6 @@ impl Tracer { } pub async fn trace(mut self, max_depth: Option) -> TraceResult { - let mut errors = vec![]; let mut seen: HashMap = HashMap::new(); let resolver = self.create_resolver(); @@ -203,17 +224,13 @@ impl Tracer { continue; } } - if let Err(err) = self - .trace_file(&resolver, file_path, file_depth, &mut seen) - .await - { - errors.push(err); - } + self.trace_file(&resolver, file_path, file_depth, &mut seen) + .await; } TraceResult { files: seen, - errors, + errors: self.errors, } } @@ -221,6 +238,8 @@ impl Tracer { let files = match globwalk::globwalk( &self.cwd, &[ + "**/*.js".parse().expect("valid glob"), + "**/*.jsx".parse().expect("valid glob"), "**/*.ts".parse().expect("valid glob"), "**/*.tsx".parse().expect("valid glob"), ], @@ -248,19 +267,29 @@ impl Tracer { let shared_self = shared_self.clone(); let resolver = resolver.clone(); futures.spawn(async move { - let (imported_files, seen_file) = - shared_self.get_imports_from_file(&resolver, &file).await?; + let mut errors = Vec::new(); + let Some((imported_files, seen_file)) = Self::get_imports_from_file( + &shared_self.source_map, + &mut errors, + &resolver, + &file, + ) + .await + else { + return (errors, None); + }; + for import in imported_files { if shared_self .files .iter() .any(|(source, _)| import.as_path() == source.as_path()) { - return Ok(Some((file, seen_file))); + return (errors, Some((file, seen_file))); } } - Ok(None) + (errors, None) }); } @@ -268,12 +297,11 @@ impl Tracer { let mut errors = Vec::new(); while let Some(result) = futures.join_next().await { - match result.unwrap() { - Ok(Some((path, file))) => { - usages.insert(path, file); - } - Ok(None) => {} - Err(err) => errors.push(err), + let (errs, file) = result.unwrap(); + errors.extend(errs); + + if let Some((path, seen_file)) = file { + usages.insert(path, seen_file); } } diff --git a/crates/turborepo/tests/query.rs b/crates/turborepo/tests/query.rs index b6767a60c5b2c..5bc3d773846fe 100644 --- a/crates/turborepo/tests/query.rs +++ b/crates/turborepo/tests/query.rs @@ -21,7 +21,7 @@ fn test_double_symlink() -> Result<(), anyhow::Error> { } #[test] -fn test_trace() -> Result<(), anyhow::Error> { +fn test_ast() -> Result<(), anyhow::Error> { // Separate because the `\\` -> `/` filter isn't compatible with ast check_json!( "turbo_trace", @@ -30,6 +30,11 @@ fn test_trace() -> Result<(), anyhow::Error> { "get `main.ts` with ast" => "query { file(path: \"main.ts\") { path ast } }", ); + Ok(()) +} + +#[test] +fn test_trace() -> Result<(), anyhow::Error> { insta::with_settings!({ filters => vec![(r"\\\\", "/")]}, { check_json!( "turbo_trace", @@ -41,8 +46,36 @@ fn test_trace() -> Result<(), anyhow::Error> { "get `circular.ts` with dependencies" => "query { file(path: \"circular.ts\") { path dependencies { files { items { path } } } } }", "get `invalid.ts` with dependencies" => "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", "get `main.ts` with depth = 0" => "query { file(path: \"main.ts\") { path dependencies(depth: 1) { files { items { path } } } } }", + "get `with_prefix.ts` with dependencies" => "query { file(path: \"with_prefix.ts\") { path dependencies { files { items { path } } } } }", ); Ok(()) }) } + +#[test] +fn test_trace_on_monorepo() -> Result<(), anyhow::Error> { + insta::with_settings!({ filters => vec![(r"\\\\", "/")]}, { + check_json!( + "turbo_trace_monorepo", + "npm@10.5.0", + "query", + "get `apps/my-app/index.ts` with dependencies" => "query { file(path: \"apps/my-app/index.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", + "get `packages/utils/index` with dependents" => "query { file(path: \"packages/utils/index.ts\") { path dependents { files { items { path } } errors { items { message } } } } }", + ); + + Ok(()) + }) +} + +#[test] +fn test_reverse_trace() -> Result<(), anyhow::Error> { + check_json!( + "turbo_trace", + "npm@10.5.0", + "query", + "get `button.tsx` with dependents" => "query { file(path: \"button.tsx\") { path dependents { files { items { path } } } } }", + ); + + Ok(()) +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..e49db2b924ed6 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,23 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "button.tsx", + "dependents": { + "files": { + "items": [ + { + "path": "invalid.ts" + }, + { + "path": "main.ts" + } + ] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap index 8354df28773f7..55436189c5f86 100644 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap @@ -17,7 +17,7 @@ expression: query_output "errors": { "items": [ { - "message": "failed to resolve import" + "message": "failed to resolve import to `./non-existent-file.js`" } ] } diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap new file mode 100644 index 0000000000000..062887b12ac53 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`with_prefix.ts`_with_dependencies_(npm@10.5.0).snap @@ -0,0 +1,23 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "with_prefix.ts", + "dependencies": { + "files": { + "items": [ + { + "path": "bar.js" + }, + { + "path": "foo.js" + } + ] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap new file mode 100644 index 0000000000000..62bfdc21cbe16 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`apps__my-app__index.ts`_with_dependencies_(npm@10.5.0).snap @@ -0,0 +1,32 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "apps/my-app/index.ts", + "dependencies": { + "files": { + "items": [ + { + "path": "apps/my-app/types.ts" + }, + { + "path": "packages/another/index.js" + }, + { + "path": "packages/utils/index.ts" + }, + { + "path": "packages/utils/my-hook.ts" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap new file mode 100644 index 0000000000000..4e6e7ed0e3329 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap @@ -0,0 +1,26 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": { + "file": { + "path": "packages/utils/index.ts", + "dependents": { + "files": { + "items": [ + { + "path": "apps/my-app/index.ts" + }, + { + "path": "packages/another/index.js" + } + ] + }, + "errors": { + "items": [] + } + } + } + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json b/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json new file mode 100644 index 0000000000000..51d884f8a82d0 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "paths": { + "@*": [ + ".*" + ] + } + } +} \ No newline at end of file diff --git a/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts b/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts new file mode 100644 index 0000000000000..b5668f3729691 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/with_prefix.ts @@ -0,0 +1 @@ +import foo from "@/foo"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore new file mode 100644 index 0000000000000..77af9fc60321d --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.turbo +.npmrc diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/.env.local b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/.env.local new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts new file mode 100644 index 0000000000000..2b6c3821b3f30 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/index.ts @@ -0,0 +1,4 @@ +import { useMyHook } from "utils/my-hook"; +import ship from "utils"; +import { blackbeard } from "../../packages/another/index.js"; +import { Pirate } from "@/types.ts"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json new file mode 100644 index 0000000000000..cf17ebf161c4a --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/package.json @@ -0,0 +1,10 @@ +{ + "name": "my-app", + "scripts": { + "build": "echo building", + "maybefails": "exit 4" + }, + "dependencies": { + "utils": "*" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json new file mode 100644 index 0000000000000..c199498e2ce9f --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "paths": { + "@/*": [ + "./*" + ] + } + } +} \ No newline at end of file diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts new file mode 100644 index 0000000000000..236ea60c41f5d --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/apps/my-app/types.ts @@ -0,0 +1,3 @@ +export interface Pirate { + ship: string; +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json new file mode 100644 index 0000000000000..83e4fa43c6849 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/package.json @@ -0,0 +1,11 @@ +{ + "name": "monorepo", + "scripts": { + "something": "turbo run build" + }, + "packageManager": "npm@10.5.0", + "workspaces": [ + "apps/**", + "packages/**" + ] +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js new file mode 100644 index 0000000000000..94639c6347160 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/index.js @@ -0,0 +1,3 @@ +import ship from "utils"; + +export const blackbeard = "Edward Teach on " + ship; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json new file mode 100644 index 0000000000000..bb796c8455b16 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/another/package.json @@ -0,0 +1,9 @@ +{ + "name": "another", + "scripts": { + "dev": "echo building" + }, + "dependencies": { + "utils": "*" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts new file mode 100644 index 0000000000000..5475301176b3b --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/index.ts @@ -0,0 +1,2 @@ +const ship = "The Queen Anne's Revenge"; +export default ship; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts new file mode 100644 index 0000000000000..dad43626ffbc4 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/my-hook.ts @@ -0,0 +1,3 @@ +export const useMyHook = () => { + console.log("arrrr matey"); +}; diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json new file mode 100644 index 0000000000000..2184abb4f4989 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/packages/utils/package.json @@ -0,0 +1,12 @@ +{ + "name": "utils", + "scripts": { + "build": "echo building", + "maybefails": "echo didnotfail" + }, + "main": "index.ts", + "exports": { + ".": "./index.ts", + "./my-hook": "./my-hook.ts" + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json new file mode 100644 index 0000000000000..9e26dfeeb6e64 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace_monorepo/turbo.json @@ -0,0 +1 @@ +{} \ No newline at end of file