From 45fe515f26eaf8c8a96dc90c5cbc2139cfa3a5e7 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 6 Sep 2023 10:52:15 -0500 Subject: [PATCH] feat: lockfile v3 (#8) --- src/error.rs | 4 +- src/lib.rs | 226 ++++++++++++++++++++++++++++++++++++---------- src/transforms.rs | 120 ++++++++++++++++++++++++ 3 files changed, 298 insertions(+), 52 deletions(-) create mode 100644 src/transforms.rs diff --git a/src/error.rs b/src/error.rs index 3613033..cadcd07 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,9 +7,9 @@ pub enum LockfileError { #[error(transparent)] Io(#[from] std::io::Error), - #[error("Unable to read lockfile: \"{0}\"")] + #[error("Unable to read lockfile. {0}")] ReadError(String), - #[error("Unable to parse contents of lockfile: \"{0}\"")] + #[error("Unable to parse contents of lockfile. {0}")] ParseError(String), } diff --git a/src/lib.rs b/src/lib.rs index 148c40c..d71ffe0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,8 @@ use serde::Deserialize; use serde::Serialize; use std::path::PathBuf; +mod transforms; + pub struct NpmPackageLockfileInfo { pub display_id: String, pub serialized_id: String, @@ -55,14 +57,18 @@ pub struct NpmPackageInfo { } #[derive(Clone, Debug, Default, Serialize, Deserialize, Hash)] -pub struct NpmContent { - /// Mapping between requests for npm packages and resolved packages, eg. +pub struct PackagesContent { + /// Mapping between requests for deno specifiers and resolved packages, eg. /// { - /// "chalk": "chalk@5.0.0", - /// "react@17": "react@17.0.1", - /// "foo@latest": "foo@1.0.0" + /// "deno:path": "deno:@std/path@1.0.0", + /// "deno:ts-morph@11": "npm:ts-morph@11.0.0", + /// "deno:@foo/bar@^2.1": "deno:@foo/bar@2.1.3", + /// "npm:@ts-morph/common@^11": "npm:@ts-morph/common@11.0.0", /// } + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + #[serde(default)] pub specifiers: BTreeMap, + /// Mapping between resolved npm specifiers and their associated info, eg. /// { /// "chalk@5.0.0": { @@ -72,36 +78,38 @@ pub struct NpmContent { /// } /// } /// } - pub packages: BTreeMap, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + #[serde(default)] + pub npm: BTreeMap, } -impl NpmContent { +impl PackagesContent { fn is_empty(&self) -> bool { - self.specifiers.is_empty() && self.packages.is_empty() + self.specifiers.is_empty() && self.npm.is_empty() } } #[derive(Debug, Clone, Serialize, Deserialize, Hash)] pub struct LockfileContent { version: String, - // have redirects at the top of the file so they're more easily auditable + // order these based on auditability + #[serde(skip_serializing_if = "PackagesContent::is_empty")] + #[serde(default)] + pub packages: PackagesContent, #[serde(skip_serializing_if = "BTreeMap::is_empty")] #[serde(default)] pub redirects: BTreeMap, /// Mapping between URLs and their checksums for "http:" and "https:" deps remote: BTreeMap, - #[serde(skip_serializing_if = "NpmContent::is_empty")] - #[serde(default)] - pub npm: NpmContent, } impl LockfileContent { fn empty() -> Self { Self { - version: "2".to_string(), - remote: BTreeMap::new(), - npm: Default::default(), + version: "3".to_string(), + packages: Default::default(), redirects: Default::default(), + remote: BTreeMap::new(), } } } @@ -166,25 +174,26 @@ impl Lockfile { }); } - let value: serde_json::Value = serde_json::from_str(content) - .map_err(|_| Error::ParseError(filename.display().to_string()))?; - let version = value.get("version").and_then(|v| v.as_str()); - let content = if version == Some("2") { - serde_json::from_value::(value) - .map_err(|_| Error::ParseError(filename.display().to_string()))? - } else { - // If there's no version field, we assume that user is using the old - // version of the lockfile. We'll migrate it in-place into v2 and it - // will be written in v2 if user uses `--lock-write` flag. - let remote: BTreeMap = serde_json::from_value(value) + let value: serde_json::Map = + serde_json::from_str(content) .map_err(|_| Error::ParseError(filename.display().to_string()))?; - LockfileContent { - version: "2".to_string(), - remote, - npm: NpmContent::default(), - redirects: Default::default(), + let version = value.get("version").and_then(|v| v.as_str()); + let value = match version { + Some("3") => value, + Some("2") => transforms::transform2_to_3(value), + None => transforms::transform2_to_3(transforms::transform1_to_2(value)), + Some(version) => { + return Err(Error::ParseError(format!( + "Unsupported lockfile version '{}'. Try upgrading Deno or recreating the lockfile.", + version + ))) } }; + let content = serde_json::from_value::(value.into()) + .map_err(|err| { + eprintln!("ERROR: {:#}", err); + Error::ParseError(filename.display().to_string()) + })?; Ok(Lockfile { overwrite, @@ -240,7 +249,7 @@ impl Lockfile { ) -> Result<(), LockfileError> { if self.overwrite { // In case --lock-write is specified check always passes - self.insert_npm(package_info); + self.insert_npm_package(package_info); Ok(()) } else { self.check_or_insert_npm(package_info) @@ -270,7 +279,7 @@ impl Lockfile { package: NpmPackageLockfileInfo, ) -> Result<(), LockfileError> { if let Some(package_info) = - self.content.npm.packages.get(&package.serialized_id) + self.content.packages.npm.get(&package.serialized_id) { if package_info.integrity.as_str() != package.integrity { return Err(LockfileError(format!( @@ -286,20 +295,20 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", ))); } } else { - self.insert_npm(package); + self.insert_npm_package(package); } Ok(()) } - fn insert_npm(&mut self, package_info: NpmPackageLockfileInfo) { + fn insert_npm_package(&mut self, package_info: NpmPackageLockfileInfo) { let dependencies = package_info .dependencies .iter() .map(|dep| (dep.name.to_string(), dep.id.to_string())) .collect::>(); - self.content.npm.packages.insert( + self.content.packages.npm.insert( package_info.serialized_id.to_string(), NpmPackageInfo { integrity: package_info.integrity, @@ -309,18 +318,22 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", self.has_content_changed = true; } - pub fn insert_npm_specifier( + pub fn insert_package_specifier( &mut self, serialized_package_req: String, serialized_package_id: String, ) { - let maybe_prev = self.content.npm.specifiers.get(&serialized_package_req); + let maybe_prev = self + .content + .packages + .specifiers + .get(&serialized_package_req); if maybe_prev.is_none() || maybe_prev != Some(&serialized_package_id) { self.has_content_changed = true; self .content - .npm + .packages .specifiers .insert(serialized_package_req, serialized_package_id); } @@ -347,14 +360,10 @@ mod tests { const LOCKFILE_JSON: &str = r#" { - "version": "2", - "remote": { - "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", - "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" - }, - "npm": { + "version": "3", + "packages": { "specifiers": {}, - "packages": { + "npm": { "nanoid@3.3.4": { "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", "dependencies": {} @@ -364,6 +373,10 @@ mod tests { "dependencies": {} } } + }, + "remote": { + "https://deno.land/std@0.71.0/textproto/mod.ts": "sha512-3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "sha512-35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" } }"#; @@ -382,6 +395,21 @@ mod tests { assert!(Lockfile::new(file_path, false).is_ok()); } + #[test] + fn future_version_unsupported() { + let file_path = PathBuf::from("lockfile.json"); + assert_eq!( + Lockfile::with_lockfile_content( + file_path, + "{ \"version\": \"2000\" }", + false + ) + .err() + .unwrap().to_string(), + "Unable to parse contents of lockfile. Unsupported lockfile version '2000'. Try upgrading Deno or recreating the lockfile.".to_string() + ); + } + #[test] fn new_valid_lockfile() { let temp_dir = TempDir::new().unwrap(); @@ -582,7 +610,7 @@ mod tests { let mut lockfile = Lockfile::with_lockfile_content( PathBuf::from("/foo/deno.lock"), r#"{ - "version": "2", + "version": "3", "redirects": { "https://deno.land/x/std/mod.ts": "https://deno.land/std@0.190.0/mod.ts" }, @@ -598,7 +626,7 @@ mod tests { assert_eq!( lockfile.as_json_string(), r#"{ - "version": "2", + "version": "3", "redirects": { "https://deno.land/x/other/mod.ts": "https://deno.land/x/other@0.1.0/mod.ts", "https://deno.land/x/std/mod.ts": "https://deno.land/std@0.190.0/mod.ts" @@ -614,7 +642,7 @@ mod tests { let mut lockfile = Lockfile::with_lockfile_content( PathBuf::from("/foo/deno.lock"), r#"{ - "version": "2", + "version": "3", "redirects": { "https://deno.land/x/std/mod.ts": "https://deno.land/std@0.190.0/mod.ts" }, @@ -640,7 +668,7 @@ mod tests { assert_eq!( lockfile.as_json_string(), r#"{ - "version": "2", + "version": "3", "redirects": { "https://deno.land/x/std/mod.ts": "https://deno.land/std@0.190.1/mod.ts", "https://deno.land/x/std/other.ts": "https://deno.land/std@0.190.1/other.ts" @@ -650,4 +678,102 @@ mod tests { "#, ); } + + #[test] + fn test_insert_deno() { + let mut lockfile = Lockfile::with_lockfile_content( + PathBuf::from("/foo/deno.lock"), + r#"{ + "version": "3", + "packages": { + "specifiers": { + "deno:path": "deno:@std/path@0.75.0" + } + }, + "remote": {} +}"#, + false, + ) + .unwrap(); + lockfile.insert_package_specifier( + "deno:path".to_string(), + "deno:@std/path@0.75.0".to_string(), + ); + assert!(!lockfile.has_content_changed); + lockfile.insert_package_specifier( + "deno:path".to_string(), + "deno:@std/path@0.75.1".to_string(), + ); + assert!(lockfile.has_content_changed); + lockfile.insert_package_specifier( + "deno:@foo/bar@^2".to_string(), + "deno:@foo/bar@2.1.2".to_string(), + ); + assert_eq!( + lockfile.as_json_string(), + r#"{ + "version": "3", + "packages": { + "specifiers": { + "deno:@foo/bar@^2": "deno:@foo/bar@2.1.2", + "deno:path": "deno:@std/path@0.75.1" + } + }, + "remote": {} +} +"#, + ); + } + + #[test] + fn read_version_1() { + let content: &str = r#"{ + "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" + }"#; + let file_path = PathBuf::from("lockfile.json"); + let lockfile = + Lockfile::with_lockfile_content(file_path, content, false).unwrap(); + assert_eq!(lockfile.content.version, "3"); + assert_eq!(lockfile.content.remote.len(), 2); + } + + #[test] + fn read_version_2() { + let content: &str = r#"{ + "version": "2", + "remote": { + "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" + }, + "npm": { + "specifiers": { + "nanoid": "nanoid@3.3.4" + }, + "packages": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + } + } + } + }"#; + let file_path = PathBuf::from("lockfile.json"); + let lockfile = + Lockfile::with_lockfile_content(file_path, content, false).unwrap(); + assert_eq!(lockfile.content.version, "3"); + assert_eq!(lockfile.content.packages.npm.len(), 2); + assert_eq!( + lockfile.content.packages.specifiers, + BTreeMap::from([( + "npm:nanoid".to_string(), + "npm:nanoid@3.3.4".to_string() + ),]) + ); + assert_eq!(lockfile.content.remote.len(), 2); + } } diff --git a/src/transforms.rs b/src/transforms.rs new file mode 100644 index 0000000..5b94ae8 --- /dev/null +++ b/src/transforms.rs @@ -0,0 +1,120 @@ +pub type JsonMap = serde_json::Map; + +pub fn transform1_to_2(json: JsonMap) -> JsonMap { + let mut new_map = JsonMap::new(); + new_map.insert("version".to_string(), "2".into()); + new_map.insert("remote".to_string(), json.into()); + new_map +} + +pub fn transform2_to_3(mut json: JsonMap) -> JsonMap { + json.insert("version".into(), "3".into()); + if let Some(serde_json::Value::Object(mut npm_obj)) = json.remove("npm") { + let mut new_obj = JsonMap::new(); + if let Some(packages) = npm_obj.remove("packages") { + new_obj.insert("npm".into(), packages); + } + if let Some(serde_json::Value::Object(specifiers)) = + npm_obj.remove("specifiers") + { + let mut new_specifiers = JsonMap::new(); + for (key, value) in specifiers { + if let serde_json::Value::String(value) = value { + new_specifiers + .insert(format!("npm:{}", key), format!("npm:{}", value).into()); + } + } + if !new_specifiers.is_empty() { + new_obj.insert("specifiers".into(), new_specifiers.into()); + } + } + json.insert("packages".into(), new_obj.into()); + } + if let Some(serde_json::Value::Object(remote_obj)) = json.get_mut("remote") { + for (_, value) in remote_obj.iter_mut() { + if let serde_json::Value::String(value) = value { + *value = format!("sha256-{}", value); + } + } + } + + json +} + +#[cfg(test)] +mod test { + use pretty_assertions::assert_eq; + use serde_json::json; + + use super::*; + + #[test] + fn test_transforms_1_to_2() { + let data: JsonMap = serde_json::from_value(json!({ + "https://github.com/": "asdf", + "https://github.com/mod.ts": "asdf2", + })) + .unwrap(); + let result = transform1_to_2(data); + assert_eq!( + result, + serde_json::from_value(json!({ + "version": "2", + "remote": { + "https://github.com/": "asdf", + "https://github.com/mod.ts": "asdf2", + } + })) + .unwrap() + ); + } + + #[test] + fn test_transforms_2_to_3() { + let data: JsonMap = serde_json::from_value(json!({ + "version": "2", + "remote": { + "https://github.com/": "asdf", + "https://github.com/mod.ts": "asdf2", + }, + "npm": { + "specifiers": { + "nanoid": "nanoid@3.3.4", + }, + "packages": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + } + } + } + })).unwrap(); + let result = transform2_to_3(data); + assert_eq!(result, serde_json::from_value(json!({ + "version": "3", + "remote": { + "https://github.com/": "sha256-asdf", + "https://github.com/mod.ts": "sha256-asdf2", + }, + "packages": { + "specifiers": { + "npm:nanoid": "npm:nanoid@3.3.4", + }, + "npm": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + } + } + } + })).unwrap()); + } +}