From 0d1eb913ba69c221bd9833200589931b42294f4b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 15 Jan 2024 16:27:23 -0500 Subject: [PATCH 01/17] Start --- Cargo.lock | 10 +++------- Cargo.toml | 5 +++++ cli/Cargo.toml | 2 +- cli/args/package_json.rs | 6 +++--- cli/factory.rs | 6 ++++++ cli/lsp/language_server.rs | 2 +- cli/npm/managed/installer.rs | 2 +- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eaf124b1ae996d..bf7d331dd60967 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1449,9 +1449,9 @@ dependencies = [ [[package]] name = "deno_lockfile" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd29f62e6dec60e585f579df3e9c2fc562aadf881319152974bc442a9042077" dependencies = [ + "deno_semver", + "indexmap", "ring", "serde", "serde_json", @@ -1577,8 +1577,6 @@ dependencies = [ [[package]] name = "deno_npm" version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718b0b55031643de7808f8b426661b22a685820f1f459e028776bcc49e07b881" dependencies = [ "anyhow", "async-trait", @@ -1671,9 +1669,7 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d3f7f5a3b2ace62b8fdede8585f5fdbd4e7dba9cb33fcaf0db54887316feaa" +version = "0.5.2" dependencies = [ "monch", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 36345b656fc584..ac7e56dabb3be5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,3 +352,8 @@ opt-level = 3 opt-level = 3 [profile.release.package.base64-simd] opt-level = 3 + +[patch.crates-io] +deno_lockfile = { path = "../deno_lockfile" } +deno_npm = { path = "../deno_npm" } +deno_semver = { path = "../deno_semver" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index faefe28c1426a7..2c481aa7d0c0bf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -66,7 +66,7 @@ deno_npm = "=0.15.3" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } # todo(dsherret): investigate https://github.com/denoland/deno_semver/commit/98f9174baef199809295077b3b68c9fa58defb9b causing # lsp_completions_auto_import_and_quick_fix_with_import_map to fail when bumping this version -deno_semver = "=0.5.1" +deno_semver = "=0.5.2" deno_task_shell = "=0.14.0" eszip = "=0.57.0" napi_sym.workspace = true diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index c304d0715c83bd..c8a3dadd1c5389 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -39,7 +39,7 @@ impl PackageJsonDepsProvider { self.0.as_ref() } - pub fn reqs(&self) -> Vec<&PackageReq> { + pub fn reqs(&self) -> Option> { match &self.0 { Some(deps) => { let mut package_reqs = deps @@ -47,9 +47,9 @@ impl PackageJsonDepsProvider { .filter_map(|r| r.as_ref().ok()) .collect::>(); package_reqs.sort(); // deterministic resolution - package_reqs + Some(package_reqs) } - None => Vec::new(), + None => None, } } } diff --git a/cli/factory.rs b/cli/factory.rs index 204ea7e87eb349..47e789f2f7be21 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -303,6 +303,12 @@ impl CliFactory { .npm_resolver .get_or_try_init_async(async { let fs = self.fs(); + + if let Some(lockfile) = self.maybe_lockfile() { + let mut lockfile = lockfile.lock(); + lockfile.set_package_json_deps(self.package_json_deps_provider().reqs()) + } + create_cli_npm_resolver(if self.options.unstable_byonm() { CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { fs: fs.clone(), diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index a3c81752675e8a..ffbe2eac9f6087 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -1176,7 +1176,7 @@ async fn create_npm_resolver( } None => CliNpmResolverManagedSnapshotOption::Specified(None), }, - // Don't provide the lockfile. We don't want these resolvers + // Don't provide the lockfile or package.json. We don't want these resolvers // updating it. Only the cache request should update the lockfile. maybe_lockfile: None, fs: Arc::new(deno_fs::RealFs), diff --git a/cli/npm/managed/installer.rs b/cli/npm/managed/installer.rs index c836ff7d87505e..f762be70ec9545 100644 --- a/cli/npm/managed/installer.rs +++ b/cli/npm/managed/installer.rs @@ -82,7 +82,7 @@ impl PackageJsonDepsInstaller { return Ok(()); // already installed by something else } - let package_reqs = inner.deps_provider.reqs(); + let package_reqs = inner.deps_provider.reqs().unwrap_or_default(); // check if something needs resolving before bothering to load all // the package information (which is slow) From 1271528473db62b33b9ecf68911e56bd9c97e521 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 17 Jan 2024 12:24:00 -0500 Subject: [PATCH 02/17] Current. --- Cargo.lock | 2 -- Cargo.toml | 1 + cli/factory.rs | 35 +++++++++++++++++++++++++++++++---- cli/graph_util.rs | 4 ++++ cli/npm/managed/resolution.rs | 17 +---------------- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf7d331dd60967..a8aa9d09390be4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1323,8 +1323,6 @@ dependencies = [ [[package]] name = "deno_graph" version = "0.63.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf16929b6a267be4bc535877303a3a36b3595b6b27543baa07d3b77b679d92" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index ac7e56dabb3be5..8b2b82ab7e270d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -357,3 +357,4 @@ opt-level = 3 deno_lockfile = { path = "../deno_lockfile" } deno_npm = { path = "../deno_npm" } deno_semver = { path = "../deno_semver" } +deno_graph = { path = "../deno_graph" } diff --git a/cli/factory.rs b/cli/factory.rs index 47e789f2f7be21..e360a1412e03a3 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -60,8 +60,11 @@ use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; +use deno_semver::jsr::JsrPackageReqReference; +use deno_semver::npm::NpmPackageReqReference; use import_map::ImportMap; use log::warn; +use std::collections::BTreeSet; use std::future::Future; use std::sync::Arc; @@ -304,9 +307,33 @@ impl CliFactory { .get_or_try_init_async(async { let fs = self.fs(); - if let Some(lockfile) = self.maybe_lockfile() { + let maybe_lockfile = self.maybe_lockfile(); + if let Some(lockfile) = maybe_lockfile.as_ref() { + // todo(THIS PR): makes no sense for this to be here + let import_map = self.maybe_import_map().await?; + let import_map_deps = import_map.as_ref().map(|import_map| { + let mut entries = BTreeSet::new(); + let import_entries = import_map + .imports() + .entries() + .chain(import_map.scopes().flat_map(|s| s.imports.entries())); + for entry in import_entries { + if let Some(value) = entry.value { + if let Ok(req_ref) = + JsrPackageReqReference::from_specifier(value) + { + entries.insert(format!("jsr:{}", req_ref.req())); + } else if let Ok(req_ref) = + NpmPackageReqReference::from_specifier(value) + { + entries.insert(format!("npm:{}", req_ref.req())); + } + } + } + entries + }); let mut lockfile = lockfile.lock(); - lockfile.set_package_json_deps(self.package_json_deps_provider().reqs()) + lockfile.set_deps(self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| s.to_string()).collect()), import_map_deps); } create_cli_npm_resolver(if self.options.unstable_byonm() { @@ -326,7 +353,7 @@ impl CliFactory { Some(snapshot) => { CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) } - None => match self.maybe_lockfile() { + None => match maybe_lockfile.as_ref() { Some(lockfile) => { CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( lockfile.clone(), @@ -335,7 +362,7 @@ impl CliFactory { None => CliNpmResolverManagedSnapshotOption::Specified(None), }, }, - maybe_lockfile: self.maybe_lockfile().as_ref().cloned(), + maybe_lockfile: maybe_lockfile.as_ref().cloned(), fs: fs.clone(), http_client: self.http_client().clone(), npm_global_cache_dir: self.deno_dir()?.npm_folder_path(), diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 53861415c62e21..342013e6cbb0ec 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -504,6 +504,10 @@ impl ModuleGraphBuilder { format!("jsr:{}", to), ); } + for (name, deps) in graph.packages.package_deps() { + lockfile + .insert_package_deps(name.to_string(), deps.map(|s| s.to_string())); + } } } diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index b020cec0388892..0f60efdb2d6663 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -388,21 +388,6 @@ fn populate_lockfile_from_snapshot( fn npm_package_to_lockfile_info( pkg: &NpmResolutionPackage, ) -> NpmPackageLockfileInfo { - fn integrity_for_lockfile( - integrity: NpmPackageVersionDistInfoIntegrity, - ) -> String { - match integrity { - NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm, - base64_hash, - } => format!("{}-{}", algorithm, base64_hash), - NpmPackageVersionDistInfoIntegrity::UnknownIntegrity(integrity) => { - integrity.to_string() - } - NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(hex) => hex.to_string(), - } - } - let dependencies = pkg .dependencies .iter() @@ -415,7 +400,7 @@ fn npm_package_to_lockfile_info( NpmPackageLockfileInfo { display_id: pkg.id.nv.to_string(), serialized_id: pkg.id.as_serialized(), - integrity: integrity_for_lockfile(pkg.dist.integrity()), + integrity: pkg.dist.integrity().for_lockfile(), dependencies, } } From 17f0643b31517836e2636e14e8d508c4a05f0cbc Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 17 Jan 2024 18:27:25 -0500 Subject: [PATCH 03/17] Workspaces support --- cli/factory.rs | 95 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 23 deletions(-) diff --git a/cli/factory.rs b/cli/factory.rs index e360a1412e03a3..be62a2ed45f0c0 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -53,6 +53,7 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::FeatureChecker; +use deno_core::serde_json; use deno_graph::GraphKind; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; @@ -301,6 +302,57 @@ impl CliFactory { pub async fn npm_resolver( &self, ) -> Result<&Arc, AnyError> { + fn imports_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + let mut items = Vec::with_capacity(obj.len()); + for value in obj.values() { + if let serde_json::Value::String(value) = value { + items.push(value); + } + } + items + } + + fn scope_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + obj.values().flat_map(|v| imports_values(Some(v))).collect() + } + + fn values_to_set<'a>( + values: impl Iterator, + ) -> BTreeSet { + let mut entries = BTreeSet::new(); + for value in values { + if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { + entries.insert(format!("jsr:{}", req_ref.req())); + } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { + entries.insert(format!("npm:{}", req_ref.req())); + } + } + entries + } + + fn import_map_value_deps( + value: &serde_json::Value, + ) -> Option> { + let obj = value.as_object()?; + let values = imports_values(obj.get("imports")) + .into_iter() + .chain(scope_values(obj.get("scopes")).into_iter()); + Some(values_to_set(values)) + } + + fn deno_json_deps(config: &deno_config::ConfigFile) -> BTreeSet { + let values = imports_values(config.json.imports.as_ref()) + .into_iter() + .chain(scope_values(config.json.scopes.as_ref()).into_iter()); + values_to_set(values) + } + self .services .npm_resolver @@ -310,30 +362,27 @@ impl CliFactory { let maybe_lockfile = self.maybe_lockfile(); if let Some(lockfile) = maybe_lockfile.as_ref() { // todo(THIS PR): makes no sense for this to be here - let import_map = self.maybe_import_map().await?; - let import_map_deps = import_map.as_ref().map(|import_map| { - let mut entries = BTreeSet::new(); - let import_entries = import_map - .imports() - .entries() - .chain(import_map.scopes().flat_map(|s| s.imports.entries())); - for entry in import_entries { - if let Some(value) = entry.value { - if let Ok(req_ref) = - JsrPackageReqReference::from_specifier(value) - { - entries.insert(format!("jsr:{}", req_ref.req())); - } else if let Ok(req_ref) = - NpmPackageReqReference::from_specifier(value) - { - entries.insert(format!("npm:{}", req_ref.req())); - } - } - } - entries - }); let mut lockfile = lockfile.lock(); - lockfile.set_deps(self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| s.to_string()).collect()), import_map_deps); + let package_json_deps = self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| s.to_string()).collect()); + let config = match self.options.maybe_workspace_config() { + Some(workspace_config) => { + deno_lockfile::WorkspaceConfig { + package_json_deps, + deps: import_map_value_deps(&workspace_config.base_import_map_value), + members: workspace_config.members.iter().map(|member| { + (member.package_name.clone(), deno_json_deps(&member.config_file)) + }).collect(), + } + } + None => { + deno_lockfile::WorkspaceConfig { + package_json_deps, + deps: self.options.maybe_config_file().as_ref().map(deno_json_deps), + members: Default::default(), + } + } + }; + lockfile.update_workspace_config(config); } create_cli_npm_resolver(if self.options.unstable_byonm() { From fd3d264c142e8e9727ce7e116f781ee0b7f4f884 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 19 Jan 2024 20:43:24 -0500 Subject: [PATCH 04/17] Updates. --- Cargo.lock | 20 ++++++++++++-------- Cargo.toml | 2 +- cli/factory.rs | 29 ++++++++++++++++++++++------- cli/npm/managed/mod.rs | 18 +++++++++++++----- cli/npm/managed/resolution.rs | 1 - 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8aa9d09390be4..f00229faa6f0fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,7 +994,7 @@ dependencies = [ "libz-sys", "log", "lsp-types", - "monch", + "monch 0.5.0", "napi_sym", "nix 0.26.2", "notify", @@ -1333,7 +1333,7 @@ dependencies = [ "import_map", "indexmap", "log", - "monch", + "monch 0.5.0", "once_cell", "parking_lot 0.12.1", "regex", @@ -1448,8 +1448,6 @@ dependencies = [ name = "deno_lockfile" version = "0.17.2" dependencies = [ - "deno_semver", - "indexmap", "ring", "serde", "serde_json", @@ -1582,7 +1580,7 @@ dependencies = [ "deno_semver", "futures", "log", - "monch", + "monch 0.5.0", "serde", "thiserror", ] @@ -1669,7 +1667,7 @@ dependencies = [ name = "deno_semver" version = "0.5.2" dependencies = [ - "monch", + "monch 0.5.0", "once_cell", "serde", "thiserror", @@ -1685,7 +1683,7 @@ dependencies = [ "anyhow", "futures", "glob", - "monch", + "monch 0.4.3", "os_pipe", "path-dedot", "tokio", @@ -3720,6 +3718,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4519a88847ba2d5ead3dc53f1060ec6a571de93f325d9c5c4968147382b1cbc3" +[[package]] +name = "monch" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52c1b33ff98142aecea13138bd399b68aa7ab5d9546c300988c345004001eea" + [[package]] name = "multimap" version = "0.8.3" @@ -6123,7 +6127,7 @@ dependencies = [ "lazy-regex", "libc", "lsp-types", - "monch", + "monch 0.5.0", "nix 0.26.2", "once_cell", "os_pipe", diff --git a/Cargo.toml b/Cargo.toml index 8b2b82ab7e270d..36d6dd394b982d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,7 +117,7 @@ libz-sys = { version = "1.1", default-features = false } log = "=0.4.20" lsp-types = "=0.94.1" # used by tower-lsp and "proposed" feature is unstable in patch releases memmem = "0.1.1" -monch = "=0.4.3" +monch = "=0.5.0" notify = "=5.0.0" num-bigint = { version = "0.4", features = ["rand"] } once_cell = "1.17.1" diff --git a/cli/factory.rs b/cli/factory.rs index be62a2ed45f0c0..d30789b6487871 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -55,6 +55,7 @@ use deno_core::FeatureChecker; use deno_core::serde_json; use deno_graph::GraphKind; +use deno_lockfile::WorkspaceMemberConfig; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; use deno_runtime::deno_node::NodeResolver; @@ -63,6 +64,7 @@ use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::PackageNv; use import_map::ImportMap; use log::warn; use std::collections::BTreeSet; @@ -363,26 +365,39 @@ impl CliFactory { if let Some(lockfile) = maybe_lockfile.as_ref() { // todo(THIS PR): makes no sense for this to be here let mut lockfile = lockfile.lock(); - let package_json_deps = self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| s.to_string()).collect()); + let package_json_deps = self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| format!("npm:{}", s)).collect()); let config = match self.options.maybe_workspace_config() { Some(workspace_config) => { deno_lockfile::WorkspaceConfig { - package_json_deps, - deps: import_map_value_deps(&workspace_config.base_import_map_value), + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: import_map_value_deps(&workspace_config.base_import_map_value), + }, members: workspace_config.members.iter().map(|member| { - (member.package_name.clone(), deno_json_deps(&member.config_file)) + (member.package_name.clone(), WorkspaceMemberConfig { + package_json_deps: None, + dependencies: Some(deno_json_deps(&member.config_file)), + }) }).collect(), } } None => { deno_lockfile::WorkspaceConfig { - package_json_deps, - deps: self.options.maybe_config_file().as_ref().map(deno_json_deps), + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: self.options.maybe_config_file().as_ref().map(deno_json_deps), + }, members: Default::default(), } } }; - lockfile.update_workspace_config(config); + lockfile.set_workspace_config(deno_lockfile::SetWorkspaceConfigOptions { + config, + nv_to_jsr_url: |nv| { + let nv = PackageNv::from_str(nv).ok()?; + Some(deno_graph::source::recommended_registry_package_url(crate::args::deno_registry_url(), &nv).to_string()) + } + }); } create_cli_npm_resolver(if self.options.unstable_byonm() { diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index b3bd77b259b896..829750c0a91694 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -225,13 +225,21 @@ async fn snapshot_from_lockfile( lockfile: Arc>, api: &dyn NpmRegistryApi, ) -> Result { - let incomplete_snapshot = { + let (incomplete_snapshot, skip_integrity_check) = { let lock = lockfile.lock(); - deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)? + ( + deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?, + lock.overwrite, + ) }; - let snapshot = - deno_npm::resolution::snapshot_from_lockfile(incomplete_snapshot, api) - .await?; + let snapshot = deno_npm::resolution::snapshot_from_lockfile( + deno_npm::resolution::SnapshotFromLockfileParams { + incomplete_snapshot, + api, + skip_integrity_check, + }, + ) + .await?; Ok(snapshot) } diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 0f60efdb2d6663..4d9c4c3e9014bf 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -10,7 +10,6 @@ use deno_core::parking_lot::RwLock; use deno_lockfile::NpmPackageDependencyLockfileInfo; use deno_lockfile::NpmPackageLockfileInfo; use deno_npm::registry::NpmPackageInfo; -use deno_npm::registry::NpmPackageVersionDistInfoIntegrity; use deno_npm::registry::NpmRegistryApi; use deno_npm::resolution::NpmPackageVersionResolutionError; use deno_npm::resolution::NpmPackagesPartitioned; From 7d5c18131bf7b752db319fab74da6bba12f95ae9 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 19 Jan 2024 20:45:43 -0500 Subject: [PATCH 05/17] Update. --- cli/lsp/language_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index ffbe2eac9f6087..a3c81752675e8a 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -1176,7 +1176,7 @@ async fn create_npm_resolver( } None => CliNpmResolverManagedSnapshotOption::Specified(None), }, - // Don't provide the lockfile or package.json. We don't want these resolvers + // Don't provide the lockfile. We don't want these resolvers // updating it. Only the cache request should update the lockfile. maybe_lockfile: None, fs: Arc::new(deno_fs::RealFs), From ae89fd8431c0908b6a2bd7ba670fe9a343e331fc Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 20 Jan 2024 19:34:38 -0500 Subject: [PATCH 06/17] Move getting deps from import map out to separate utility function --- Cargo.lock | 2 +- Cargo.toml | 2 +- cli/Cargo.toml | 2 +- cli/factory.rs | 63 ++++-------------------------------------- cli/util/import_map.rs | 61 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f00229faa6f0fa..88d3f71ea89b93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1665,7 +1665,7 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.5.2" +version = "0.5.3" dependencies = [ "monch 0.5.0", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 36d6dd394b982d..b609e7bf7e8157 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -356,5 +356,5 @@ opt-level = 3 [patch.crates-io] deno_lockfile = { path = "../deno_lockfile" } deno_npm = { path = "../deno_npm" } -deno_semver = { path = "../deno_semver" } deno_graph = { path = "../deno_graph" } +deno_semver = { path = "../deno_semver" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2c481aa7d0c0bf..82ed9156852684 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -66,7 +66,7 @@ deno_npm = "=0.15.3" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } # todo(dsherret): investigate https://github.com/denoland/deno_semver/commit/98f9174baef199809295077b3b68c9fa58defb9b causing # lsp_completions_auto_import_and_quick_fix_with_import_map to fail when bumping this version -deno_semver = "=0.5.2" +deno_semver = "=0.5.3" deno_task_shell = "=0.14.0" eszip = "=0.57.0" napi_sym.workspace = true diff --git a/cli/factory.rs b/cli/factory.rs index d30789b6487871..c6e61cf4f548c2 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -44,6 +44,8 @@ use crate::standalone::DenoCompileBinaryWriter; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path_maybe_not_exists; +use crate::util::import_map::deno_json_deps; +use crate::util::import_map::import_map_deps; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use crate::worker::CliMainWorkerFactory; @@ -53,7 +55,6 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::FeatureChecker; -use deno_core::serde_json; use deno_graph::GraphKind; use deno_lockfile::WorkspaceMemberConfig; use deno_runtime::deno_fs; @@ -62,12 +63,9 @@ use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; -use deno_semver::jsr::JsrPackageReqReference; -use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use import_map::ImportMap; use log::warn; -use std::collections::BTreeSet; use std::future::Future; use std::sync::Arc; @@ -304,57 +302,6 @@ impl CliFactory { pub async fn npm_resolver( &self, ) -> Result<&Arc, AnyError> { - fn imports_values(value: Option<&serde_json::Value>) -> Vec<&String> { - let Some(obj) = value.and_then(|v| v.as_object()) else { - return Vec::new(); - }; - let mut items = Vec::with_capacity(obj.len()); - for value in obj.values() { - if let serde_json::Value::String(value) = value { - items.push(value); - } - } - items - } - - fn scope_values(value: Option<&serde_json::Value>) -> Vec<&String> { - let Some(obj) = value.and_then(|v| v.as_object()) else { - return Vec::new(); - }; - obj.values().flat_map(|v| imports_values(Some(v))).collect() - } - - fn values_to_set<'a>( - values: impl Iterator, - ) -> BTreeSet { - let mut entries = BTreeSet::new(); - for value in values { - if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { - entries.insert(format!("jsr:{}", req_ref.req())); - } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { - entries.insert(format!("npm:{}", req_ref.req())); - } - } - entries - } - - fn import_map_value_deps( - value: &serde_json::Value, - ) -> Option> { - let obj = value.as_object()?; - let values = imports_values(obj.get("imports")) - .into_iter() - .chain(scope_values(obj.get("scopes")).into_iter()); - Some(values_to_set(values)) - } - - fn deno_json_deps(config: &deno_config::ConfigFile) -> BTreeSet { - let values = imports_values(config.json.imports.as_ref()) - .into_iter() - .chain(scope_values(config.json.scopes.as_ref()).into_iter()); - values_to_set(values) - } - self .services .npm_resolver @@ -371,12 +318,12 @@ impl CliFactory { deno_lockfile::WorkspaceConfig { root: WorkspaceMemberConfig { package_json_deps, - dependencies: import_map_value_deps(&workspace_config.base_import_map_value), + dependencies: Some(import_map_deps(&workspace_config.base_import_map_value).into_iter().map(|req| req.to_string()).collect()), }, members: workspace_config.members.iter().map(|member| { (member.package_name.clone(), WorkspaceMemberConfig { package_json_deps: None, - dependencies: Some(deno_json_deps(&member.config_file)), + dependencies: Some(deno_json_deps(&member.config_file).into_iter().map(|req| req.to_string()).collect()), }) }).collect(), } @@ -385,7 +332,7 @@ impl CliFactory { deno_lockfile::WorkspaceConfig { root: WorkspaceMemberConfig { package_json_deps, - dependencies: self.options.maybe_config_file().as_ref().map(deno_json_deps), + dependencies: self.options.maybe_config_file().as_ref().map(|config| deno_json_deps(config).into_iter().map(|req| req.to_string()).collect()), }, members: Default::default(), } diff --git a/cli/util/import_map.rs b/cli/util/import_map.rs index 0b78a133c3d8e1..b1e1ba10a94d6d 100644 --- a/cli/util/import_map.rs +++ b/cli/util/import_map.rs @@ -1,17 +1,78 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::collections::HashSet; + use deno_ast::ParsedSource; use deno_core::error::AnyError; +use deno_core::serde_json; use deno_core::ModuleSpecifier; use deno_graph::DefaultModuleAnalyzer; use deno_graph::DependencyDescriptor; use deno_graph::DynamicTemplatePart; use deno_graph::MediaType; use deno_graph::TypeScriptReference; +use deno_semver::jsr::JsrPackageReqReference; +use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::JsrOrNpmPackageReq; use import_map::ImportMap; use crate::graph_util::format_range_with_colors; +pub fn import_map_deps( + value: &serde_json::Value, +) -> HashSet { + let Some(obj) = value.as_object() else { + return Default::default(); + }; + let values = imports_values(obj.get("imports")) + .into_iter() + .chain(scope_values(obj.get("scopes")).into_iter()); + values_to_set(values) +} + +pub fn deno_json_deps( + config: &deno_config::ConfigFile, +) -> HashSet { + let values = imports_values(config.json.imports.as_ref()) + .into_iter() + .chain(scope_values(config.json.scopes.as_ref()).into_iter()); + values_to_set(values) +} + +fn imports_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + let mut items = Vec::with_capacity(obj.len()); + for value in obj.values() { + if let serde_json::Value::String(value) = value { + items.push(value); + } + } + items +} + +fn scope_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + obj.values().flat_map(|v| imports_values(Some(v))).collect() +} + +fn values_to_set<'a>( + values: impl Iterator, +) -> HashSet { + let mut entries = HashSet::new(); + for value in values { + if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { + entries.insert(JsrOrNpmPackageReq::jsr(req_ref.into_inner().req)); + } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { + entries.insert(JsrOrNpmPackageReq::npm(req_ref.into_inner().req)); + } + } + entries +} + pub struct ImportMapUnfurler<'a> { import_map: &'a ImportMap, } From 287f08a3ed3990104c29bdc0f7cdf7b88fbefc86 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 20 Jan 2024 20:27:00 -0500 Subject: [PATCH 07/17] Fix lsp issue with latest deno_semver changes --- Cargo.lock | 2 -- Cargo.toml | 1 + cli/Cargo.toml | 2 -- cli/lsp/analysis.rs | 29 +++++++++++++++++++---------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88d3f71ea89b93..6b4a01abb1e6dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3156,8 +3156,6 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" [[package]] name = "import_map" version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1624c269d2ca7427d79471c8ba799abe9215e706cc0182d7b86fc856a35d565b" dependencies = [ "indexmap", "log", diff --git a/Cargo.toml b/Cargo.toml index b609e7bf7e8157..f6c1eb56b93360 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -358,3 +358,4 @@ deno_lockfile = { path = "../deno_lockfile" } deno_npm = { path = "../deno_npm" } deno_graph = { path = "../deno_graph" } deno_semver = { path = "../deno_semver" } +import_map = { path = "../import_map/rs-lib" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 82ed9156852684..043f75d7dfefc4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -64,8 +64,6 @@ deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm = "=0.15.3" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } -# todo(dsherret): investigate https://github.com/denoland/deno_semver/commit/98f9174baef199809295077b3b68c9fa58defb9b causing -# lsp_completions_auto_import_and_quick_fix_with_import_map to fail when bumping this version deno_semver = "=0.5.3" deno_task_shell = "=0.14.0" eszip = "=0.57.0" diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 5301fbeea00adc..3789fc8d3bc846 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -25,12 +25,14 @@ use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::NpmResolver; use deno_runtime::deno_node::PathClean; use deno_runtime::permissions::PermissionsContainer; +use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; use import_map::ImportMap; use once_cell::sync::Lazy; use regex::Regex; use std::cmp::Ordering; use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use tower_lsp::lsp_types as lsp; use tower_lsp::lsp_types::Position; @@ -211,17 +213,24 @@ impl<'a> TsResponseImportMapper<'a> { if !pkg_reqs.is_empty() { let sub_path = self.resolve_package_path(specifier); if let Some(import_map) = self.maybe_import_map { - for pkg_req in &pkg_reqs { - let paths = vec![ - concat_npm_specifier("npm:", pkg_req, sub_path.as_deref()), - concat_npm_specifier("npm:/", pkg_req, sub_path.as_deref()), - ]; - for path in paths { - if let Some(mapped_path) = ModuleSpecifier::parse(&path) - .ok() - .and_then(|s| import_map.lookup(&s, referrer)) + let pkg_reqs = pkg_reqs.iter().collect::>(); + for entry in import_map.entries_for_referrer(&referrer) { + if let Some(value) = entry.raw_value { + if let Ok(package_ref) = + NpmPackageReqReference::from_str(value) { - return Some(mapped_path); + if pkg_reqs.contains(package_ref.req()) { + let sub_path = sub_path.as_deref().unwrap_or("/"); + let right_sub_path = + package_ref.sub_path().unwrap_or("/"); + let left_sub_path = sub_path + .strip_prefix(right_sub_path) + .unwrap_or(sub_path); + return Some(format!( + "{}{}", + entry.raw_key, left_sub_path + )); + } } } } From 643381e52239bbec0f4adcaa37b38b73863d9d2a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 20 Jan 2024 20:37:30 -0500 Subject: [PATCH 08/17] Update to `JsrDepPackageReq` from `JsrOrNpmPackageReq` --- cli/util/import_map.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cli/util/import_map.rs b/cli/util/import_map.rs index b1e1ba10a94d6d..a7900a6c71ab2d 100644 --- a/cli/util/import_map.rs +++ b/cli/util/import_map.rs @@ -11,16 +11,14 @@ use deno_graph::DependencyDescriptor; use deno_graph::DynamicTemplatePart; use deno_graph::MediaType; use deno_graph::TypeScriptReference; +use deno_semver::jsr::JsrDepPackageReq; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; -use deno_semver::package::JsrOrNpmPackageReq; use import_map::ImportMap; use crate::graph_util::format_range_with_colors; -pub fn import_map_deps( - value: &serde_json::Value, -) -> HashSet { +pub fn import_map_deps(value: &serde_json::Value) -> HashSet { let Some(obj) = value.as_object() else { return Default::default(); }; @@ -32,7 +30,7 @@ pub fn import_map_deps( pub fn deno_json_deps( config: &deno_config::ConfigFile, -) -> HashSet { +) -> HashSet { let values = imports_values(config.json.imports.as_ref()) .into_iter() .chain(scope_values(config.json.scopes.as_ref()).into_iter()); @@ -61,13 +59,13 @@ fn scope_values(value: Option<&serde_json::Value>) -> Vec<&String> { fn values_to_set<'a>( values: impl Iterator, -) -> HashSet { +) -> HashSet { let mut entries = HashSet::new(); for value in values { if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { - entries.insert(JsrOrNpmPackageReq::jsr(req_ref.into_inner().req)); + entries.insert(JsrDepPackageReq::jsr(req_ref.into_inner().req)); } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { - entries.insert(JsrOrNpmPackageReq::npm(req_ref.into_inner().req)); + entries.insert(JsrDepPackageReq::npm(req_ref.into_inner().req)); } } entries From 1a59e45c2df3fd13f63b5db26163be1b467f0ee8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 20 Jan 2024 22:11:44 -0500 Subject: [PATCH 09/17] Fix up mapping npm to import map bare specifier --- cli/lsp/analysis.rs | 41 +++++++++++---- cli/tests/integration/lsp_tests.rs | 51 +++++++++++++++++++ .../1.0.0/dist/entry-a.js | 2 +- .../1.0.0/dist/entry-b.d.ts | 1 + .../1.0.0/dist/entry-b.js | 3 ++ .../types-exports-subpaths/1.0.0/package.json | 3 ++ 6 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.d.ts create mode 100644 cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.js diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 3789fc8d3bc846..ef6c58aa04a4e5 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -214,26 +214,31 @@ impl<'a> TsResponseImportMapper<'a> { let sub_path = self.resolve_package_path(specifier); if let Some(import_map) = self.maybe_import_map { let pkg_reqs = pkg_reqs.iter().collect::>(); + let mut matches = Vec::new(); for entry in import_map.entries_for_referrer(&referrer) { if let Some(value) = entry.raw_value { if let Ok(package_ref) = NpmPackageReqReference::from_str(value) { if pkg_reqs.contains(package_ref.req()) { - let sub_path = sub_path.as_deref().unwrap_or("/"); - let right_sub_path = - package_ref.sub_path().unwrap_or("/"); - let left_sub_path = sub_path - .strip_prefix(right_sub_path) - .unwrap_or(sub_path); - return Some(format!( - "{}{}", - entry.raw_key, left_sub_path - )); + let sub_path = sub_path.as_deref().unwrap_or(""); + let value_sub_path = package_ref.sub_path().unwrap_or(""); + if let Some(key_sub_path) = + sub_path.strip_prefix(value_sub_path) + { + matches + .push(format!("{}{}", entry.raw_key, key_sub_path)); + } } } } } + + // select the shortest match + matches.sort_by(|a, b| a.len().cmp(&b.len())); + if let Some(matched) = matches.first() { + return Some(matched.to_string()); + } } // if not found in the import map, return the first pkg req @@ -276,6 +281,22 @@ impl<'a> TsResponseImportMapper<'a> { // a search for the .d.ts file instead if specifier_path.extension().and_then(|e| e.to_str()) == Some("js") { search_paths.insert(0, specifier_path.with_extension("d.ts")); + } else if let Some(file_name) = + specifier_path.file_name().and_then(|f| f.to_str()) + { + // In some other cases, typescript will provide the .d.ts extension, but the + // export might not have a .d.ts defined. In that case, look for the corresponding + // JavaScript file after not being able to find the .d.ts file. + if let Some(file_stem) = file_name.strip_suffix(".d.ts") { + search_paths + .push(specifier_path.with_file_name(format!("{}.js", file_stem))); + } else if let Some(file_stem) = file_name.strip_suffix(".d.cts") { + search_paths + .push(specifier_path.with_file_name(format!("{}.cjs", file_stem))); + } else if let Some(file_stem) = file_name.strip_suffix(".d.mts") { + search_paths + .push(specifier_path.with_file_name(format!("{}.mjs", file_stem))); + } } for search_path in search_paths { diff --git a/cli/tests/integration/lsp_tests.rs b/cli/tests/integration/lsp_tests.rs index 018dc93304911a..206fadfdf3d41e 100644 --- a/cli/tests/integration/lsp_tests.rs +++ b/cli/tests/integration/lsp_tests.rs @@ -6450,6 +6450,7 @@ fn lsp_completions_auto_import_and_quick_fix_with_import_map() { "imports": { "print_hello": "http://localhost:4545/subdir/print_hello.ts", "chalk": "npm:chalk@~5", + "nested/": "npm:/@denotest/types-exports-subpaths@1/nested/", "types-exports-subpaths/": "npm:/@denotest/types-exports-subpaths@1/" } }"#; @@ -6471,6 +6472,7 @@ fn lsp_completions_auto_import_and_quick_fix_with_import_map() { "import chalk from 'npm:chalk@~5';\n", "import chalk from 'npm:chalk@~5';\n", "import {printHello} from 'print_hello';\n", + "import {entryB} from 'npm:@denotest/types-exports-subpaths@1/nested/entry-b';\n", "\n", ), } @@ -6483,6 +6485,7 @@ fn lsp_completions_auto_import_and_quick_fix_with_import_map() { "arguments": [ [ "npm:@denotest/types-exports-subpaths@1/client", + "npm:@denotest/types-exports-subpaths@1/nested/entry-b", "npm:chalk@^5.0", "npm:chalk@~5", "http://localhost:4545/subdir/print_hello.ts", @@ -6822,6 +6825,54 @@ fn lsp_completions_auto_import_and_quick_fix_with_import_map() { } }]) ); + + // try auto-import with npm package with sub-path on value side of import map + client.did_open(json!({ + "textDocument": { + "uri": "file:///a/nested_path.ts", + "languageId": "typescript", + "version": 1, + "text": "entry", + } + })); + let list = client.get_completion_list( + "file:///a/nested_path.ts", + (0, 5), + json!({ "triggerKind": 1 }), + ); + assert!(!list.is_incomplete); + let item = list + .items + .iter() + .find(|item| item.label == "entryB") + .unwrap(); + + let res = client.write_request("completionItem/resolve", item); + assert_eq!( + res, + json!({ + "label": "entryB", + "labelDetails": { + "description": "nested/entry-b", + }, + "kind": 3, + "detail": "function entryB(): \"b\"", + "documentation": { + "kind": "markdown", + "value": "" + }, + "sortText": "￿16_0", + "additionalTextEdits": [ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 0 } + }, + "newText": "import { entryB } from \"nested/entry-b\";\n\n" + } + ] + }) + ); } #[test] diff --git a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-a.js b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-a.js index 070b1ccbd63672..84f1f2c94152de 100644 --- a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-a.js +++ b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-a.js @@ -1,3 +1,3 @@ -export function entryC() { +export function entryA() { return 12; } diff --git a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.d.ts b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.d.ts new file mode 100644 index 00000000000000..382d1995ee4777 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.d.ts @@ -0,0 +1 @@ +export function entryB(): "b"; diff --git a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.js b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.js new file mode 100644 index 00000000000000..162d4f1908fa91 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/dist/entry-b.js @@ -0,0 +1,3 @@ +export function entryB() { + return "b"; +} diff --git a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/package.json index 169017560997fa..cc43cf2ed7d6d9 100644 --- a/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/package.json +++ b/cli/tests/testdata/npm/registry/@denotest/types-exports-subpaths/1.0.0/package.json @@ -18,6 +18,9 @@ }, "./entry-a": { "import": "./dist/entry-a.js" + }, + "./nested/entry-b": { + "import": "./dist/entry-b.js" } } } From 013b0651ebd1430621b6e2bf7ef837a85d3bf219 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 09:21:57 -0500 Subject: [PATCH 10/17] Fix clippy --- cli/lsp/analysis.rs | 4 ++-- cli/util/import_map.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index ef6c58aa04a4e5..7339964c71d266 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -215,7 +215,7 @@ impl<'a> TsResponseImportMapper<'a> { if let Some(import_map) = self.maybe_import_map { let pkg_reqs = pkg_reqs.iter().collect::>(); let mut matches = Vec::new(); - for entry in import_map.entries_for_referrer(&referrer) { + for entry in import_map.entries_for_referrer(referrer) { if let Some(value) = entry.raw_value { if let Ok(package_ref) = NpmPackageReqReference::from_str(value) @@ -235,7 +235,7 @@ impl<'a> TsResponseImportMapper<'a> { } // select the shortest match - matches.sort_by(|a, b| a.len().cmp(&b.len())); + matches.sort_by_key(|a| a.len()); if let Some(matched) = matches.first() { return Some(matched.to_string()); } diff --git a/cli/util/import_map.rs b/cli/util/import_map.rs index a7900a6c71ab2d..10c5dc3f42669f 100644 --- a/cli/util/import_map.rs +++ b/cli/util/import_map.rs @@ -24,7 +24,7 @@ pub fn import_map_deps(value: &serde_json::Value) -> HashSet { }; let values = imports_values(obj.get("imports")) .into_iter() - .chain(scope_values(obj.get("scopes")).into_iter()); + .chain(scope_values(obj.get("scopes"))); values_to_set(values) } @@ -33,7 +33,7 @@ pub fn deno_json_deps( ) -> HashSet { let values = imports_values(config.json.imports.as_ref()) .into_iter() - .chain(scope_values(config.json.scopes.as_ref()).into_iter()); + .chain(scope_values(config.json.scopes.as_ref())); values_to_set(values) } From b550130e6a005aef99090df5d5ca758541ab4927 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 11:25:13 -0500 Subject: [PATCH 11/17] Updates for deno_lockfile --- cli/args/mod.rs | 4 + cli/args/package_json.rs | 28 +----- cli/factory.rs | 2 + cli/standalone/binary.rs | 10 -- cli/tests/integration/run_tests.rs | 143 +++++++++++++++++++++++++++++ test_util/src/fs.rs | 13 +++ 6 files changed, 163 insertions(+), 37 deletions(-) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index ae48aae50e94e5..d73f0ba6071838 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1336,6 +1336,10 @@ impl CliOptions { self.flags.no_npm } + pub fn no_config(&self) -> bool { + self.flags.config_flag == deno_config::ConfigFlag::Disabled + } + pub fn permissions_options(&self) -> PermissionsOptions { PermissionsOptions { allow_env: self.flags.allow_env.clone(), diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index c8a3dadd1c5389..67481fd07a5820 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_npm::registry::parse_dep_entry_name_and_raw_version; -use deno_npm::registry::PackageDepNpmSchemeValueParseError; use deno_runtime::deno_node::PackageJson; use deno_semver::package::PackageReq; use deno_semver::VersionReq; @@ -16,8 +15,6 @@ use thiserror::Error; #[derive(Debug, Error, Clone)] pub enum PackageJsonDepValueParseError { - #[error(transparent)] - SchemeValue(#[from] PackageDepNpmSchemeValueParseError), #[error(transparent)] Specifier(#[from] VersionReqSpecifierParseError), #[error("Not implemented scheme '{scheme}'")] @@ -77,9 +74,7 @@ pub fn get_local_package_json_version_reqs( scheme: value.split(':').next().unwrap().to_string(), }); } - let (name, version_req) = parse_dep_entry_name_and_raw_version(key, value) - .map_err(PackageJsonDepValueParseError::SchemeValue)?; - + let (name, version_req) = parse_dep_entry_name_and_raw_version(key, value); let result = VersionReq::parse_from_specifier(version_req); match result { Ok(version_req) => Ok(PackageReq { @@ -159,27 +154,6 @@ mod test { use super::*; - #[test] - fn test_parse_dep_entry_name_and_raw_version() { - let cases = [ - ("test", "^1.2", Ok(("test", "^1.2"))), - ("test", "1.x - 2.6", Ok(("test", "1.x - 2.6"))), - ("test", "npm:package@^1.2", Ok(("package", "^1.2"))), - ( - "test", - "npm:package", - Err("Could not find @ symbol in npm url 'npm:package'"), - ), - ]; - for (key, value, expected_result) in cases { - let result = parse_dep_entry_name_and_raw_version(key, value); - match result { - Ok(result) => assert_eq!(result, expected_result.unwrap()), - Err(err) => assert_eq!(err.to_string(), expected_result.err().unwrap()), - } - } - } - fn get_local_package_json_version_reqs_for_tests( package_json: &PackageJson, ) -> IndexMap> { diff --git a/cli/factory.rs b/cli/factory.rs index c6e61cf4f548c2..c881902da44938 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -339,6 +339,8 @@ impl CliFactory { } }; lockfile.set_workspace_config(deno_lockfile::SetWorkspaceConfigOptions { + no_npm: self.options.no_npm(), + no_config: self.options.no_config(), config, nv_to_jsr_url: |nv| { let nv = PackageNv::from_str(nv).ok()?; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 18880b7a16a7e8..0d24cb1e67ebe8 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -18,7 +18,6 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; -use deno_npm::registry::PackageDepNpmSchemeValueParseError; use deno_npm::NpmSystemInfo; use deno_runtime::permissions::PermissionsOptions; use deno_semver::package::PackageReq; @@ -50,7 +49,6 @@ const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; #[derive(Serialize, Deserialize)] enum SerializablePackageJsonDepValueParseError { - SchemeValue(String), Specifier(String), Unsupported { scheme: String }, } @@ -58,9 +56,6 @@ enum SerializablePackageJsonDepValueParseError { impl SerializablePackageJsonDepValueParseError { pub fn from_err(err: PackageJsonDepValueParseError) -> Self { match err { - PackageJsonDepValueParseError::SchemeValue(err) => { - Self::SchemeValue(err.value) - } PackageJsonDepValueParseError::Specifier(err) => { Self::Specifier(err.source.to_string()) } @@ -72,11 +67,6 @@ impl SerializablePackageJsonDepValueParseError { pub fn into_err(self) -> PackageJsonDepValueParseError { match self { - SerializablePackageJsonDepValueParseError::SchemeValue(value) => { - PackageJsonDepValueParseError::SchemeValue( - PackageDepNpmSchemeValueParseError { value }, - ) - } SerializablePackageJsonDepValueParseError::Specifier(source) => { PackageJsonDepValueParseError::Specifier( VersionReqSpecifierParseError { diff --git a/cli/tests/integration/run_tests.rs b/cli/tests/integration/run_tests.rs index 8db06470d19371..b494f6543db50e 100644 --- a/cli/tests/integration/run_tests.rs +++ b/cli/tests/integration/run_tests.rs @@ -1020,6 +1020,149 @@ fn lock_redirects() { ); } +#[test] +fn lock_deno_json_package_json_deps() { + let context = TestContextBuilder::new() + .use_temp_cwd() + .use_http_server() + .add_npm_env_vars() + .add_jsr_env_vars() + .build(); + let temp_dir = context.temp_dir().path(); + let deno_json = temp_dir.join("deno.json"); + let package_json = temp_dir.join("package.json"); + + // add a jsr and npm dependency + deno_json.write_json(&json!({ + "imports": { + "esm-basic": "npm:@denotest/esm-basic", + "module_graph": "jsr:@denotest/module_graph@1.4", + } + })); + let main_ts = temp_dir.join("main.ts"); + main_ts.write("import 'esm-basic'; import 'module_graph';"); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + let lockfile = temp_dir.join("deno.lock"); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + }, + "npm": { + "@denotest/esm-basic@1.0.0": { + "integrity": "sha512-d4xM3xFg7CXOg2CVsDAU7Q6pKvyFz1/MJpMjaZYsOnJifm9bRbH6nhg77gNqsTpF9RDkourrjKxLAXae4FSvXQ==", + "dependencies": {} + } + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4", + "npm:@denotest/esm-basic" + ] + } + })); + + // now remove the npm dependency from the deno.json and move + // it to a package.json that uses an alias + deno_json.write_json(&json!({ + "imports": { + "module_graph": "jsr:@denotest/module_graph@1.4", + } + })); + package_json.write_json(&json!({ + "dependencies": { + "esm-basic": "npm:@denotest/esm-basic" + } + })); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + main_ts.write("import 'module_graph';"); + context + .new_command() + // ensure this doesn't clear out packageJson below + .args("cache --no-npm main.ts") + .run() + .skip_output_check(); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + }, + "npm": { + "@denotest/esm-basic@1.0.0": { + "integrity": "sha512-d4xM3xFg7CXOg2CVsDAU7Q6pKvyFz1/MJpMjaZYsOnJifm9bRbH6nhg77gNqsTpF9RDkourrjKxLAXae4FSvXQ==", + "dependencies": {} + } + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4" + ], + "packageJson": { + "dependencies": [ + "npm:@denotest/esm-basic" + ] + } + } + })); + + // now remove the package.json + package_json.remove_file(); + + // cache and it will remove the package.json + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4" + ] + } + })); + + // now remove the deno.json + deno_json.remove_file(); + + lockfile.assert_matches_json(json!({ + "version": "3", + "remote": {} + })); +} + itest!(mts_dmts_mjs { args: "run subdir/import.mts", output: "run/mts_dmts_mjs.out", diff --git a/test_util/src/fs.rs b/test_util/src/fs.rs index 9cfbcaaa70716b..17620276bcdf83 100644 --- a/test_util/src/fs.rs +++ b/test_util/src/fs.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use pretty_assertions::assert_eq; use std::borrow::Cow; use std::ffi::OsStr; use std::fs; @@ -218,6 +219,7 @@ impl PathRef { } } + #[track_caller] pub fn assert_matches_file(&self, wildcard_file: impl AsRef) -> &Self { let wildcard_file = testdata_path().join(wildcard_file); println!("output path {}", wildcard_file); @@ -225,11 +227,22 @@ impl PathRef { self.assert_matches_text(&expected_text) } + #[track_caller] pub fn assert_matches_text(&self, wildcard_text: impl AsRef) -> &Self { let actual = self.read_to_string(); assert_wildcard_match(&actual, wildcard_text.as_ref()); self } + + #[track_caller] + pub fn assert_matches_json(&self, expected: serde_json::Value) { + let actual_json = self.read_json_value(); + if actual_json != expected { + let actual_text = serde_json::to_string_pretty(&actual_json).unwrap(); + let expected_text = serde_json::to_string_pretty(&expected).unwrap(); + assert_eq!(actual_text, expected_text); + } + } } #[cfg(not(windows))] From 35cf391bc0ac99f76178b919a380b2d41e0b1328 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 14:11:14 -0500 Subject: [PATCH 12/17] Updates --- cli/factory.rs | 128 +++++++++++++-------- cli/main.rs | 10 +- cli/tests/integration/npm_tests.rs | 44 +++---- cli/tests/integration/run_tests.rs | 10 +- cli/tests/testdata/npm/lock_file/main.out | 3 + cli/tests/testdata/run/lock_check_ok2.json | 22 ++-- 6 files changed, 136 insertions(+), 81 deletions(-) diff --git a/cli/factory.rs b/cli/factory.rs index c881902da44938..993b25c0b8f0b5 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -293,10 +293,84 @@ impl CliFactory { } pub fn maybe_lockfile(&self) -> &Option>> { - self - .services - .lockfile - .get_or_init(|| self.options.maybe_lockfile()) + self.services.lockfile.get_or_init(|| { + let maybe_lockfile = self.options.maybe_lockfile(); + + // initialize the lockfile with the workspace's configuration + if let Some(lockfile) = &maybe_lockfile { + let package_json_deps = self + .package_json_deps_provider() + .reqs() + .map(|reqs| reqs.into_iter().map(|s| format!("npm:{}", s)).collect()) + .unwrap_or_default(); + let mut lockfile = lockfile.lock(); + let config = match self.options.maybe_workspace_config() { + Some(workspace_config) => deno_lockfile::WorkspaceConfig { + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: import_map_deps( + &workspace_config.base_import_map_value, + ) + .into_iter() + .map(|req| req.to_string()) + .collect(), + }, + members: workspace_config + .members + .iter() + .map(|member| { + ( + member.package_name.clone(), + WorkspaceMemberConfig { + package_json_deps: Default::default(), + dependencies: deno_json_deps(&member.config_file) + .into_iter() + .map(|req| req.to_string()) + .collect(), + }, + ) + }) + .collect(), + }, + None => deno_lockfile::WorkspaceConfig { + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: self + .options + .maybe_config_file() + .as_ref() + .map(|config| { + deno_json_deps(config) + .into_iter() + .map(|req| req.to_string()) + .collect() + }) + .unwrap_or_default(), + }, + members: Default::default(), + }, + }; + lockfile.set_workspace_config( + deno_lockfile::SetWorkspaceConfigOptions { + no_npm: self.options.no_npm(), + no_config: self.options.no_config(), + config, + nv_to_jsr_url: |nv| { + let nv = PackageNv::from_str(nv).ok()?; + Some( + deno_graph::source::recommended_registry_package_url( + crate::args::deno_registry_url(), + &nv, + ) + .to_string(), + ) + }, + }, + ); + } + + maybe_lockfile + }) } pub async fn npm_resolver( @@ -307,48 +381,6 @@ impl CliFactory { .npm_resolver .get_or_try_init_async(async { let fs = self.fs(); - - let maybe_lockfile = self.maybe_lockfile(); - if let Some(lockfile) = maybe_lockfile.as_ref() { - // todo(THIS PR): makes no sense for this to be here - let mut lockfile = lockfile.lock(); - let package_json_deps = self.package_json_deps_provider().reqs().map(|reqs| reqs.into_iter().map(|s| format!("npm:{}", s)).collect()); - let config = match self.options.maybe_workspace_config() { - Some(workspace_config) => { - deno_lockfile::WorkspaceConfig { - root: WorkspaceMemberConfig { - package_json_deps, - dependencies: Some(import_map_deps(&workspace_config.base_import_map_value).into_iter().map(|req| req.to_string()).collect()), - }, - members: workspace_config.members.iter().map(|member| { - (member.package_name.clone(), WorkspaceMemberConfig { - package_json_deps: None, - dependencies: Some(deno_json_deps(&member.config_file).into_iter().map(|req| req.to_string()).collect()), - }) - }).collect(), - } - } - None => { - deno_lockfile::WorkspaceConfig { - root: WorkspaceMemberConfig { - package_json_deps, - dependencies: self.options.maybe_config_file().as_ref().map(|config| deno_json_deps(config).into_iter().map(|req| req.to_string()).collect()), - }, - members: Default::default(), - } - } - }; - lockfile.set_workspace_config(deno_lockfile::SetWorkspaceConfigOptions { - no_npm: self.options.no_npm(), - no_config: self.options.no_config(), - config, - nv_to_jsr_url: |nv| { - let nv = PackageNv::from_str(nv).ok()?; - Some(deno_graph::source::recommended_registry_package_url(crate::args::deno_registry_url(), &nv).to_string()) - } - }); - } - create_cli_npm_resolver(if self.options.unstable_byonm() { CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { fs: fs.clone(), @@ -366,7 +398,7 @@ impl CliFactory { Some(snapshot) => { CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) } - None => match maybe_lockfile.as_ref() { + None => match self.maybe_lockfile().as_ref() { Some(lockfile) => { CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( lockfile.clone(), @@ -375,7 +407,7 @@ impl CliFactory { None => CliNpmResolverManagedSnapshotOption::Specified(None), }, }, - maybe_lockfile: maybe_lockfile.as_ref().cloned(), + maybe_lockfile: self.maybe_lockfile().as_ref().cloned(), fs: fs.clone(), http_client: self.http_client().clone(), npm_global_cache_dir: self.deno_dir()?.npm_folder_path(), diff --git a/cli/main.rs b/cli/main.rs index 53c7bdf5a769e5..bd93017531894a 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -38,6 +38,7 @@ use deno_core::error::AnyError; use deno_core::error::JsError; use deno_core::futures::FutureExt; use deno_core::unsync::JoinHandle; +use deno_npm::resolution::SnapshotFromLockfileError; use deno_runtime::colors; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics; @@ -261,7 +262,14 @@ fn unwrap_or_exit(result: Result) -> T { if let Some(e) = error.downcast_ref::() { error_string = format_js_error(e); - } else if let Some(e) = error.downcast_ref::() { + } else if let Some(args::LockfileError::IntegrityCheckFailed(e)) = + error.downcast_ref::() + { + error_string = e.to_string(); + error_code = 10; + } else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) = + error.downcast_ref::() + { error_string = e.to_string(); error_code = 10; } diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index cfb1861b78b736..194c45d47919d4 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -256,7 +256,7 @@ itest!(import_map { http_server: true, }); -itest!(lock_file { +itest!(lock_file_integrity_failure { args: "run --allow-read --allow-env --lock npm/lock_file/lock.json npm/lock_file/main.js", output: "npm/lock_file/main.out", envs: env_vars_for_npm_tests(), @@ -1517,10 +1517,9 @@ fn lock_file_lock_write() { #[test] fn auto_discover_lock_file() { - let _server = http_server(); + let context = TestContextBuilder::for_npm().use_temp_cwd().build(); - let deno_dir = util::new_deno_dir(); - let temp_dir = util::TempDir::new(); + let temp_dir = context.temp_dir(); // write empty config file temp_dir.write("deno.json", "{}"); @@ -1541,25 +1540,26 @@ fn auto_discover_lock_file() { }"#; temp_dir.write("deno.lock", lock_file_content); - let deno = util::deno_cmd_with_deno_dir(&deno_dir) - .current_dir(temp_dir.path()) - .arg("run") - .arg("--unstable") - .arg("-A") - .arg("npm:@denotest/bin/cli-esm") - .arg("test") - .envs(env_vars_for_npm_tests()) - .piped_output() - .spawn() - .unwrap(); - let output = deno.wait_with_output().unwrap(); - assert!(!output.status.success()); - assert_eq!(output.status.code(), Some(10)); + let output = context + .new_command() + .args("run --unstable -A npm:@denotest/bin/cli-esm test") + .run(); + output + .assert_matches_text( +r#"Download http://localhost:4545/npm/registry/@denotest/bin +error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package +is the same as when the lockfile was generated. - let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains( - "Integrity check failed for npm package: \"@denotest/bin@1.0.0\"" - )); +Actual: sha512-28ndZLyo7AMfeSwgDGzsKnsb9ObeH+vVDKGsSJMd3MibMTKbls7ozY5EWFrCi1vaLAg20P9kWhN06/w74T0gMg== +Expected: sha512-foobar + +This could be caused by: + * the lock file may be corrupt + * the source itself may be corrupt + +Use "--lock-write" flag to regenerate the lockfile at "[WILDCARD]deno.lock". +"#) + .assert_exit_code(10); } #[test] diff --git a/cli/tests/integration/run_tests.rs b/cli/tests/integration/run_tests.rs index b494f6543db50e..0a2c4bfabefaae 100644 --- a/cli/tests/integration/run_tests.rs +++ b/cli/tests/integration/run_tests.rs @@ -1154,8 +1154,14 @@ fn lock_deno_json_package_json_deps() { } })); - // now remove the deno.json - deno_json.remove_file(); + // now remove the deps from the deno.json + deno_json.write("{}"); + main_ts.write(""); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); lockfile.assert_matches_json(json!({ "version": "3", diff --git a/cli/tests/testdata/npm/lock_file/main.out b/cli/tests/testdata/npm/lock_file/main.out index b8447bee630753..65e881be6a924c 100644 --- a/cli/tests/testdata/npm/lock_file/main.out +++ b/cli/tests/testdata/npm/lock_file/main.out @@ -2,6 +2,9 @@ Download [WILDCARD] error: Integrity check failed for npm package: "@babel/parser@7.19.0". Unable to verify that the package is the same as when the lockfile was generated. +Actual: sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== +Expected: sha512-foobar! + This could be caused by: * the lock file may be corrupt * the source itself may be corrupt diff --git a/cli/tests/testdata/run/lock_check_ok2.json b/cli/tests/testdata/run/lock_check_ok2.json index 162c755e2c4d88..96a84a69005308 100644 --- a/cli/tests/testdata/run/lock_check_ok2.json +++ b/cli/tests/testdata/run/lock_check_ok2.json @@ -1,10 +1,16 @@ { - "http://localhost:4545/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_application_x_typescript.t4.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_typescript.t1.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_video_mp2t.t3.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_video_vdn.t2.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18" + "version": "3", + "remote": { + "http://localhost:4545/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_application_x_typescript.t4.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_typescript.t1.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_video_mp2t.t3.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_video_vdn.t2.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18" + }, + "workspace": { + "dependencies": [] + } } From de790fa819bcc2a29ce8bca8c4c679991740ebcd Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 14:41:41 -0500 Subject: [PATCH 13/17] Bump deps. --- Cargo.lock | 8 ++++++-- Cargo.toml | 2 -- cli/Cargo.toml | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b4a01abb1e6dd..4fc79f840f559e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1665,7 +1665,9 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.5.3" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49e14effd9df8ed261f7a1a34ac19bbaf0fa940c59bd19a6d8313cf41525e1c" dependencies = [ "monch 0.5.0", "once_cell", @@ -3155,7 +3157,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" [[package]] name = "import_map" -version = "0.18.1" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556779a5de0289679854a5344c22f466698fd2661f7170db90df7fa5013b281e" dependencies = [ "indexmap", "log", diff --git a/Cargo.toml b/Cargo.toml index f6c1eb56b93360..c826cc07d9b498 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -357,5 +357,3 @@ opt-level = 3 deno_lockfile = { path = "../deno_lockfile" } deno_npm = { path = "../deno_npm" } deno_graph = { path = "../deno_graph" } -deno_semver = { path = "../deno_semver" } -import_map = { path = "../import_map/rs-lib" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 043f75d7dfefc4..53dec22c595578 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -64,7 +64,7 @@ deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm = "=0.15.3" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_semver = "=0.5.3" +deno_semver = "=0.5.4" deno_task_shell = "=0.14.0" eszip = "=0.57.0" napi_sym.workspace = true @@ -97,7 +97,7 @@ flate2.workspace = true fs3.workspace = true glob = "0.3.1" hex.workspace = true -import_map = { version = "=0.18.1", features = ["ext"] } +import_map = { version = "=0.18.2", features = ["ext"] } indexmap.workspace = true jsonc-parser = { version = "=0.23.0", features = ["serde"] } lazy-regex.workspace = true From 7fd6ddf4d4fb2e4855159d6ca0170deacd2dcd6d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 17:19:26 -0500 Subject: [PATCH 14/17] Update some deps --- Cargo.lock | 8 ++++++-- Cargo.toml | 4 +--- cli/Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 755315b451021b..2da10849b2bf0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.63.3" +version = "0.63.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be751db3da720bcd35d42f9c0967a04cafd52d554d79262426d1c00179b0c085" dependencies = [ "anyhow", "async-trait", @@ -1453,7 +1455,9 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.17.2" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe06eda519ed05b69da567bcba1d728c482fd553ddaa2ffe008468158da6de0" dependencies = [ "ring", "serde", diff --git a/Cargo.toml b/Cargo.toml index bd80fc9fb21fb9..51ab37bb577b36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ deno_runtime = { version = "0.140.0", path = "./runtime" } napi_sym = { version = "0.62.0", path = "./cli/napi/sym" } deno_bench_util = { version = "0.126.0", path = "./bench_util" } test_util = { path = "./test_util" } -deno_lockfile = "0.17.2" +deno_lockfile = "0.18.0" deno_media_type = { version = "0.1.1", features = ["module_specifier"] } denokv_proto = "0.5.0" @@ -354,6 +354,4 @@ opt-level = 3 opt-level = 3 [patch.crates-io] -deno_lockfile = { path = "../deno_lockfile" } deno_npm = { path = "../deno_npm" } -deno_graph = { path = "../deno_graph" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 3e578c05d44238..cddc150a3b184d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -59,7 +59,7 @@ deno_config = "=0.8.0" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.93.0", features = ["html"] } deno_emit = "=0.33.0" -deno_graph = "=0.63.3" +deno_graph = "=0.63.4" deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm = "=0.15.3" From c0aeedaf3ada6f09b9d8691e7c9e768ffb5a3374 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 17:48:29 -0500 Subject: [PATCH 15/17] Update deps --- Cargo.lock | 34 ++++++++++++++-------------------- Cargo.toml | 3 --- cli/Cargo.toml | 4 ++-- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2ad3f969dc6e3..e13539ee5ca32a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,7 +992,7 @@ dependencies = [ "libz-sys", "log", "lsp-types", - "monch 0.5.0", + "monch", "napi_sym", "nix 0.26.2", "notify", @@ -1329,9 +1329,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.63.3" +version = "0.63.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf16929b6a267be4bc535877303a3a36b3595b6b27543baa07d3b77b679d92" +checksum = "be751db3da720bcd35d42f9c0967a04cafd52d554d79262426d1c00179b0c085" dependencies = [ "anyhow", "async-trait", @@ -1342,7 +1342,7 @@ dependencies = [ "import_map", "indexmap", "log", - "monch 0.4.3", + "monch", "once_cell", "parking_lot 0.12.1", "regex", @@ -1455,9 +1455,9 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd29f62e6dec60e585f579df3e9c2fc562aadf881319152974bc442a9042077" +checksum = "dfe06eda519ed05b69da567bcba1d728c482fd553ddaa2ffe008468158da6de0" dependencies = [ "ring", "serde", @@ -1583,9 +1583,9 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.15.3" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718b0b55031643de7808f8b426661b22a685820f1f459e028776bcc49e07b881" +checksum = "376262760b173ff01f8f5d05d58a64f6d863472396afb5582590fa0949342854" dependencies = [ "anyhow", "async-trait", @@ -1593,7 +1593,7 @@ dependencies = [ "deno_semver", "futures", "log", - "monch 0.4.3", + "monch", "serde", "thiserror", ] @@ -1682,7 +1682,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b49e14effd9df8ed261f7a1a34ac19bbaf0fa940c59bd19a6d8313cf41525e1c" dependencies = [ - "monch 0.5.0", + "monch", "once_cell", "serde", "thiserror", @@ -1698,7 +1698,7 @@ dependencies = [ "anyhow", "futures", "glob", - "monch 0.5.0", + "monch", "os_pipe", "path-dedot", "tokio", @@ -2325,9 +2325,9 @@ dependencies = [ [[package]] name = "eszip" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081e749cb42d7e52b9a066564b86c70e586614d5de684080a2be861f35a0721" +checksum = "214f3a524473bb2c76385c02fdada6c2791ba5e07e9ac72a6c1e373d665b1680" dependencies = [ "anyhow", "base64", @@ -3727,12 +3727,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "monch" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4519a88847ba2d5ead3dc53f1060ec6a571de93f325d9c5c4968147382b1cbc3" - [[package]] name = "monch" version = "0.5.0" @@ -6095,7 +6089,7 @@ dependencies = [ "lazy-regex", "libc", "lsp-types", - "monch 0.5.0", + "monch", "nix 0.26.2", "once_cell", "os_pipe", diff --git a/Cargo.toml b/Cargo.toml index 51ab37bb577b36..8cb389fc2f9558 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,6 +352,3 @@ opt-level = 3 opt-level = 3 [profile.release.package.base64-simd] opt-level = 3 - -[patch.crates-io] -deno_npm = { path = "../deno_npm" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b586953a66a4fa..3edea3f3424ba6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -62,11 +62,11 @@ deno_emit = "=0.33.0" deno_graph = "=0.63.4" deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.15.3" +deno_npm = "=0.16.0" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver = "=0.5.4" deno_task_shell = "=0.14.3" -eszip = "=0.57.0" +eszip = "=0.58.0" napi_sym.workspace = true async-trait.workspace = true From cd753ccab12ce40c1c68ad2bf786d32c8f47a3df Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 17:55:42 -0500 Subject: [PATCH 16/17] Update --- cli/tests/integration/lsp_tests.rs | 1 - cli/tests/testdata/run/lock_check_ok2.json | 3 --- 2 files changed, 4 deletions(-) diff --git a/cli/tests/integration/lsp_tests.rs b/cli/tests/integration/lsp_tests.rs index dd918847e80c3c..2fc3d29e82bace 100644 --- a/cli/tests/integration/lsp_tests.rs +++ b/cli/tests/integration/lsp_tests.rs @@ -6473,7 +6473,6 @@ fn lsp_completions_auto_import_and_quick_fix_with_import_map() { "import chalk from 'npm:chalk@~5';\n", "import {entryB} from 'npm:@denotest/types-exports-subpaths@1/nested/entry-b';\n", "import {printHello} from 'print_hello';\n", - "import {entryB} from 'npm:@denotest/types-exports-subpaths@1/nested/entry-b';\n", "\n", ), } diff --git a/cli/tests/testdata/run/lock_check_ok2.json b/cli/tests/testdata/run/lock_check_ok2.json index 96a84a69005308..14d8b711711aef 100644 --- a/cli/tests/testdata/run/lock_check_ok2.json +++ b/cli/tests/testdata/run/lock_check_ok2.json @@ -9,8 +9,5 @@ "http://localhost:4545/subdir/mt_text_typescript.t1.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", "http://localhost:4545/subdir/mt_video_mp2t.t3.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", "http://localhost:4545/subdir/mt_video_vdn.t2.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18" - }, - "workspace": { - "dependencies": [] } } From a4bb7417602e7f85e8b044d0abcebdf43c9bb0f5 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 21 Jan 2024 18:17:12 -0500 Subject: [PATCH 17/17] Fix tests for linux/mac --- cli/tests/integration/npm_tests.rs | 2 +- cli/tests/integration/run_tests.rs | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 194c45d47919d4..9840f27713960a 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1550,7 +1550,7 @@ r#"Download http://localhost:4545/npm/registry/@denotest/bin error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package is the same as when the lockfile was generated. -Actual: sha512-28ndZLyo7AMfeSwgDGzsKnsb9ObeH+vVDKGsSJMd3MibMTKbls7ozY5EWFrCi1vaLAg20P9kWhN06/w74T0gMg== +Actual: sha512-[WILDCARD] Expected: sha512-foobar This could be caused by: diff --git a/cli/tests/integration/run_tests.rs b/cli/tests/integration/run_tests.rs index fbed40988b48d6..d8f238fa7d45d7 100644 --- a/cli/tests/integration/run_tests.rs +++ b/cli/tests/integration/run_tests.rs @@ -1047,6 +1047,21 @@ fn lock_deno_json_package_json_deps() { .run() .skip_output_check(); let lockfile = temp_dir.join("deno.lock"); + // todo(dsherret): it would be nice if the test server didn't produce + // different hashes depending on what operating system it's running on + let esm_basic_integrity = lockfile + .read_json_value() + .get("packages") + .unwrap() + .get("npm") + .unwrap() + .get("@denotest/esm-basic@1.0.0") + .unwrap() + .get("integrity") + .unwrap() + .as_str() + .unwrap() + .to_string(); lockfile.assert_matches_json(json!({ "version": "3", "packages": { @@ -1056,7 +1071,7 @@ fn lock_deno_json_package_json_deps() { }, "npm": { "@denotest/esm-basic@1.0.0": { - "integrity": "sha512-d4xM3xFg7CXOg2CVsDAU7Q6pKvyFz1/MJpMjaZYsOnJifm9bRbH6nhg77gNqsTpF9RDkourrjKxLAXae4FSvXQ==", + "integrity": esm_basic_integrity, "dependencies": {} } } @@ -1106,7 +1121,7 @@ fn lock_deno_json_package_json_deps() { }, "npm": { "@denotest/esm-basic@1.0.0": { - "integrity": "sha512-d4xM3xFg7CXOg2CVsDAU7Q6pKvyFz1/MJpMjaZYsOnJifm9bRbH6nhg77gNqsTpF9RDkourrjKxLAXae4FSvXQ==", + "integrity": esm_basic_integrity, "dependencies": {} } }