From ac362f97ecd8a7edb9e92cf7d9e5533de6c06175 Mon Sep 17 00:00:00 2001 From: Christian Duerr Date: Thu, 23 Nov 2023 19:28:55 +0100 Subject: [PATCH 1/3] Improve yarn v1 parser This fixes multiple issues in the v1 yarn parser: - Add support for path dependencies - Add support for git:// dependencies - Fix aliased dependencies - Ignore http(s) dependencies Closes #1299. --- lockfile/src/javascript.rs | 62 ++++++++-- lockfile/src/lib.rs | 2 +- lockfile/src/parsers/yarn.rs | 88 +++++++++++--- tests/fixtures/yarn-v1.lock | 26 +++- tests/fixtures/yarn-v1.trailing_newlines.lock | 115 ------------------ 5 files changed, 144 insertions(+), 149 deletions(-) delete mode 100644 tests/fixtures/yarn-v1.trailing_newlines.lock diff --git a/lockfile/src/javascript.rs b/lockfile/src/javascript.rs index 2ff650814..9b550bdb2 100644 --- a/lockfile/src/javascript.rs +++ b/lockfile/src/javascript.rs @@ -499,23 +499,59 @@ mod tests { #[test] fn lock_parse_yarn_v1() { - for p in [ - include_str!("../../tests/fixtures/yarn-v1.lock"), - include_str!("../../tests/fixtures/yarn-v1.trailing_newlines.lock"), - ] { - let pkgs = YarnLock.parse(p).unwrap(); + let with_trailing_newlines = include_str!("../../tests/fixtures/yarn-v1.lock"); + let without_trailing_newlines = format!("{}\n", with_trailing_newlines.trim_end()); - assert_eq!(pkgs.len(), 17); + for p in [with_trailing_newlines, &without_trailing_newlines] { + let pkgs = YarnLock.parse(p).unwrap(); - assert_eq!(pkgs[0].name, "@yarnpkg/lockfile"); - assert_eq!(pkgs[0].version, PackageVersion::FirstParty("1.1.0".into())); + assert_eq!(pkgs.len(), 20); - assert_eq!(pkgs[3].name, "cliui"); - assert_eq!(pkgs[3].version, PackageVersion::FirstParty("7.0.4".into())); + let expected_pkgs = [ + Package { + name: "@yarnpkg/lockfile".into(), + version: PackageVersion::FirstParty("1.1.0".into()), + package_type: PackageType::Npm, + }, + Package { + name: "cliui".into(), + version: PackageVersion::FirstParty("7.0.4".into()), + package_type: PackageType::Npm, + }, + Package { + name: "yargs".into(), + version: PackageVersion::FirstParty("16.2.0".into()), + package_type: PackageType::Npm, + }, + Package { + name: "strip-ansi".into(), + version: PackageVersion::FirstParty("6.0.1".into()), + package_type: PackageType::Npm, + }, + Package { + name: "test".into(), + version: PackageVersion::Path(Some("../test".into())), + package_type: PackageType::Npm, + }, + Package { + name: "quoted_path".into(), + version: PackageVersion::Path(Some("../quoted_path".into())), + package_type: PackageType::Npm, + }, + Package { + name: "imaginary".into(), + version: PackageVersion::Git( + "git://github.com/phylum-dev/imaginary#\ + 2a00da2067b7017f769c9100205a2a5f267a884b" + .into(), + ), + package_type: PackageType::Npm, + }, + ]; - let last = pkgs.last().unwrap(); - assert_eq!(last.name, "yargs"); - assert_eq!(last.version, PackageVersion::FirstParty("16.2.0".into())); + for expected_pkg in expected_pkgs { + assert!(pkgs.contains(&expected_pkg), "missing package {expected_pkg:?}"); + } } } diff --git a/lockfile/src/lib.rs b/lockfile/src/lib.rs index 702000238..2d427c42a 100644 --- a/lockfile/src/lib.rs +++ b/lockfile/src/lib.rs @@ -577,7 +577,7 @@ mod tests { #[test] fn parsers_only_parse_their_lockfiles() { for (format, lockfile_count) in [ - (LockfileFormat::Yarn, 4), + (LockfileFormat::Yarn, 3), (LockfileFormat::Npm, 2), (LockfileFormat::Pnpm, 1), (LockfileFormat::Gem, 1), diff --git a/lockfile/src/parsers/yarn.rs b/lockfile/src/parsers/yarn.rs index 9ee68fee8..9830132c1 100644 --- a/lockfile/src/parsers/yarn.rs +++ b/lockfile/src/parsers/yarn.rs @@ -1,3 +1,6 @@ +//! Yaml v1 lockfile parser. + +use nom::bytes::complete::take_till; use nom::InputTakeAtPosition; use phylum_types::types::package::PackageType; @@ -12,14 +15,17 @@ pub fn parse(input: &str) -> IResult<&str, Vec> { return Ok(("", Vec::new())); } - many1(entry)(input) + let (input, packages) = many1(entry)(input)?; + let filtered = packages.into_iter().flatten().collect(); + + Ok((input, filtered)) } fn yarn_lock_header(input: &str) -> IResult<&str, &str> { recognize(opt(tuple((count(take_till_line_end, 2), multispace0))))(input) } -fn entry(input: &str) -> IResult<&str, Package> { +fn entry(input: &str) -> IResult<&str, Option> { let (i, capture) = recognize(many_till( take_till_line_end, recognize(tuple((space0, alt((line_ending, eof))))), @@ -29,28 +35,80 @@ fn entry(input: &str) -> IResult<&str, Package> { Ok((i, my_entry)) } -fn parse_entry(input: &str) -> IResult<&str, Package> { - context("entry", tuple((entry_name, entry_version)))(input).map(|(next_input, res)| { - let (name, version) = res; - (next_input, Package { - name: name.to_string(), - version: PackageVersion::FirstParty(version.to_string()), - package_type: PackageType::Npm, - }) - }) +fn parse_entry(input: &str) -> IResult<&str, Option> { + let (input, (name, version)) = context("entry", tuple((entry_name, entry_version)))(input)?; + + let version = match version { + Some(version) => version, + None => return Ok((input, None)), + }; + + let package = Package { version, name: name.to_string(), package_type: PackageType::Npm }; + + Ok((input, Some(package))) } fn entry_name(input: &str) -> IResult<&str, &str> { + // Strip optional quotes. let (i, _) = opt(tag(r#"""#))(input)?; + + // Strip optional aliased package name. + let (i, _) = recognize(opt(tuple((take_until("@npm:"), tag("@npm:")))))(i)?; + + // Allow for up to one leading `@` in package name (like `@angular/cli`). let opt_at = opt(tag("@")); - let name = tuple((opt_at, take_until("@"))); - context("name", recognize(name))(i) + + // Consume everything until version separator as name. + let name_parser = tuple((opt_at, take_until("@"))); + context("name", recognize(name_parser))(i) } -fn entry_version(input: &str) -> IResult<&str, &str> { +fn entry_version(input: &str) -> IResult<&str, Option> { + // Handle path dependencies. + if input.starts_with("@./") || input.starts_with("@../") || input.starts_with("@/") { + return path_dep(input); + } + + // Handle git dependencies. + if input.starts_with("@git://") { + return git_dep(input); + } + + // Ignore HTTP(S) dependencies. + // + // These could be either git or tar dependencies, so to avoid miscategorization + // we just ignore them. + if input.starts_with("@http://") || input.starts_with("@https://") { + return Ok((input, None)); + } + + // Parse version field. let (i, _) = take_until(r#"version"#)(input)?; let version_key = tuple((tag(r#"version"#), opt(tag(r#"""#)), tag(r#" ""#))); - context("version", delimited(version_key, is_version, tag(r#"""#)))(i) + let version_parser = delimited(version_key, is_version, tag(r#"""#)); + let (i, version) = context("version", version_parser)(i)?; + + let package_version = PackageVersion::FirstParty(version.to_string()); + + Ok((i, Some(package_version))) +} + +fn path_dep(input: &str) -> IResult<&str, Option> { + let (input, _) = tag("@")(input)?; + let (input, path) = take_till(|c| matches!(c, '"' | ',' | ':'))(input)?; + let package_version = PackageVersion::Path(Some(path.into())); + Ok((input, Some(package_version))) +} + +fn git_dep(input: &str) -> IResult<&str, Option> { + // Parse resolved field. + let (input, _) = take_until(r#"resolved"#)(input)?; + let (input, _) = tuple((tag(r#"resolved"#), opt(tag(r#"""#)), tag(r#" ""#)))(input)?; + let (input, url) = take_until("\"")(input)?; + + let package_version = PackageVersion::Git(url.into()); + + Ok((input, Some(package_version))) } fn is_version(input: &str) -> IResult<&str, &str> { diff --git a/tests/fixtures/yarn-v1.lock b/tests/fixtures/yarn-v1.lock index 619401628..cafc0f34f 100644 --- a/tests/fixtures/yarn-v1.lock +++ b/tests/fixtures/yarn-v1.lock @@ -74,12 +74,12 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - ansi-regex "^5.0.0" + ansi-regex "^5.0.1" wrap-ansi@^7.0.0: version "7.0.0" @@ -100,6 +100,9 @@ yargs-parser@^20.2.2: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.7.tgz#61df85c113edfb5a7a4e36eb8aa60ef423cbc90a" integrity sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw== +test@../test: + version "1.0.0" + yargs@^16.2.0: version "16.2.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" @@ -112,3 +115,16 @@ yargs@^16.2.0: string-width "^4.2.0" y18n "^5.0.5" yargs-parser "^20.2.2" + +"express@https://github.com/expressjs/express": + version "4.18.2" + resolved "https://github.com/expressjs/express#2a00da2067b7017f769c9100205a2a5f267a884b" + +"imaginary@git://github.com/phylum-dev/imaginary": + version "6.7.8" + resolved "git://github.com/phylum-dev/imaginary#2a00da2067b7017f769c9100205a2a5f267a884b" + +"quoted_path@../quoted_path": + version "1.0.0" + + diff --git a/tests/fixtures/yarn-v1.trailing_newlines.lock b/tests/fixtures/yarn-v1.trailing_newlines.lock deleted file mode 100644 index d6f4ec0b5..000000000 --- a/tests/fixtures/yarn-v1.trailing_newlines.lock +++ /dev/null @@ -1,115 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@yarnpkg/lockfile@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" - integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ== - -ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== - -ansi-styles@^4.0.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" - integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== - dependencies: - color-convert "^2.0.1" - -cliui@^7.0.2: - version "7.0.4" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" - integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== - dependencies: - string-width "^4.2.0" - strip-ansi "^6.0.0" - wrap-ansi "^7.0.0" - -color-convert@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" - integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== - dependencies: - color-name "~1.1.4" - -color-name@~1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" - integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== - -emoji-regex@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" - integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== - -escalade@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" - integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== - -get-caller-file@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" - integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== - -is-fullwidth-code-point@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" - integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== - -require-directory@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" - integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= - -string-width@^4.1.0, string-width@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" - integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" - -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== - dependencies: - ansi-regex "^5.0.0" - -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -y18n@^5.0.5: - version "5.0.8" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" - integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== - -yargs-parser@^20.2.2: - version "20.2.7" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.7.tgz#61df85c113edfb5a7a4e36eb8aa60ef423cbc90a" - integrity sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw== - -yargs@^16.2.0: - version "16.2.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" - integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== - dependencies: - cliui "^7.0.2" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.0" - y18n "^5.0.5" - yargs-parser "^20.2.2" - From f8db83655b07955c8ba7ad3512a1fbc262222ea1 Mon Sep 17 00:00:00 2001 From: Christian Duerr Date: Mon, 27 Nov 2023 23:21:54 +0100 Subject: [PATCH 2/3] Fix minor style issues --- lockfile/src/parsers/yarn.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lockfile/src/parsers/yarn.rs b/lockfile/src/parsers/yarn.rs index 9830132c1..90bacbcc4 100644 --- a/lockfile/src/parsers/yarn.rs +++ b/lockfile/src/parsers/yarn.rs @@ -1,4 +1,4 @@ -//! Yaml v1 lockfile parser. +//! Yarn v1 lockfile parser. use nom::bytes::complete::take_till; use nom::InputTakeAtPosition; @@ -26,13 +26,13 @@ fn yarn_lock_header(input: &str) -> IResult<&str, &str> { } fn entry(input: &str) -> IResult<&str, Option> { - let (i, capture) = recognize(many_till( + let (input, capture) = recognize(many_till( take_till_line_end, recognize(tuple((space0, alt((line_ending, eof))))), ))(input)?; let (_, my_entry) = parse_entry(capture)?; - Ok((i, my_entry)) + Ok((input, my_entry)) } fn parse_entry(input: &str) -> IResult<&str, Option> { @@ -50,17 +50,17 @@ fn parse_entry(input: &str) -> IResult<&str, Option> { fn entry_name(input: &str) -> IResult<&str, &str> { // Strip optional quotes. - let (i, _) = opt(tag(r#"""#))(input)?; + let (input, _) = opt(tag(r#"""#))(input)?; // Strip optional aliased package name. - let (i, _) = recognize(opt(tuple((take_until("@npm:"), tag("@npm:")))))(i)?; + let (input, _) = recognize(opt(tuple((take_until("@npm:"), tag("@npm:")))))(input)?; // Allow for up to one leading `@` in package name (like `@angular/cli`). let opt_at = opt(tag("@")); // Consume everything until version separator as name. let name_parser = tuple((opt_at, take_until("@"))); - context("name", recognize(name_parser))(i) + context("name", recognize(name_parser))(input) } fn entry_version(input: &str) -> IResult<&str, Option> { @@ -83,14 +83,14 @@ fn entry_version(input: &str) -> IResult<&str, Option> { } // Parse version field. - let (i, _) = take_until(r#"version"#)(input)?; + let (input, _) = take_until(r#"version"#)(input)?; let version_key = tuple((tag(r#"version"#), opt(tag(r#"""#)), tag(r#" ""#))); let version_parser = delimited(version_key, is_version, tag(r#"""#)); - let (i, version) = context("version", version_parser)(i)?; + let (input, version) = context("version", version_parser)(input)?; let package_version = PackageVersion::FirstParty(version.to_string()); - Ok((i, Some(package_version))) + Ok((input, Some(package_version))) } fn path_dep(input: &str) -> IResult<&str, Option> { From 98b47930784bfeeca65e87d62022d48b2314d81c Mon Sep 17 00:00:00 2001 From: Christian Duerr Date: Mon, 27 Nov 2023 23:22:34 +0100 Subject: [PATCH 3/3] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a324bcdaa..d0cfbe8db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Aliased dependency names in `package-lock.json` +- Aliased dependency names in `yarn.lock` ## [5.8.1] - 2023-11-07