Add go.mod parser (#1413)

* Add go.mod parser * Update docs * Update changelog * Fix typo Co-authored-by: Christian Dürr <[email protected]> * PR changes --------- Co-authored-by: Christian Dürr <[email protected]>
phylum-dev · May 8, 2024 · 7eb96a0 · 7eb96a0
1 parent 7305a6a
commit 7eb96a0
Show file tree

Hide file tree

Showing 10 changed files with 432 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Added
 
 - PNPM v9 lockfile support
+- Support for parsing `go.mod` files with a Go directive of version 1.17 and higher
 
 ## Changed
 

diff --git a/docs/commands/phylum_analyze.md b/docs/commands/phylum_analyze.md
@@ -27,7 +27,7 @@ Usage: phylum analyze [OPTIONS] [DEPENDENCY_FILE]...
 
 `-t`, `--type` `<TYPE>`
 &emsp; Dependency file type used for all lockfiles (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `gomod`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
 
 `--skip-sandbox`
 &emsp; Run lockfile generation without sandbox protection

diff --git a/docs/commands/phylum_init.md b/docs/commands/phylum_init.md
@@ -21,7 +21,7 @@ Usage: phylum init [OPTIONS] [PROJECT_NAME]
 
 `-t`, `--type` `<TYPE>`
 &emsp; Dependency file type used for all lockfiles (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `gomod`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
 
 `-f`, `--force`
 &emsp; Overwrite existing configurations without confirmation

diff --git a/docs/commands/phylum_parse.md b/docs/commands/phylum_parse.md
@@ -15,7 +15,7 @@ Usage: phylum parse [OPTIONS] [DEPENDENCY_FILE]...
 
 `-t`, `--type` `<TYPE>`
 &emsp; Dependency file type used for all lockfiles (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `pnpm`, `gem`, `pip`, `poetry`, `pipenv`, `mvn`, `gradle`, `nugetlock`, `msbuild`, `gomod`, `go`, `cargo`, `spdx`, `cyclonedx`, `auto`
 
 `--skip-sandbox`
 &emsp; Run lockfile generation without sandbox protection

diff --git a/docs/supported_lockfiles.md b/docs/supported_lockfiles.md
@@ -16,6 +16,7 @@ The Phylum CLI supports processing many different lockfiles:
 | `mvn`         | `effective-pom.xml`                                                    |
 | `gradle`      | `gradle.lockfile`                                                      |
 | `go`          | `go.sum`                                                               |
+| `gomod`       | `go.mod`                                                               |
 | `cargo`       | `Cargo.lock`                                                           |
 | `spdx`        | `*.spdx.json` <br /> `*.spdx.yaml` <br /> `*.spdx.yml` <br /> `*.spdx` |
 | `cyclonedx`   | `*bom.json` <br /> `*bom.xml`                                          |

diff --git a/lockfile/src/golang.rs b/lockfile/src/golang.rs
@@ -9,7 +9,7 @@ use lockfile_generator::Generator;
 use nom::error::convert_error;
 use nom::Finish;
 
-use crate::parsers::go_sum;
+use crate::parsers::{go_mod, go_sum};
 use crate::{Package, Parse};
 
 pub struct GoSum;
@@ -37,6 +37,50 @@ impl Parse for GoSum {
     }
 }
 
+pub struct GoDeps {
+    pub go: String,
+    pub modules: Vec<Package>,
+}
+
+fn check_go_directive(version: &str) -> anyhow::Result<()> {
+    let mut parts = version.split(|c: char| !c.is_numeric());
+    if let (Some(major), Some(minor)) = (parts.next(), parts.next()) {
+        let major: u32 = major.parse().unwrap_or(0);
+        let minor: u32 = minor.parse().unwrap_or(0);
+
+        // Check if version meets the criteria.
+        if major < 1 || (major == 1 && minor < 17) {
+            return Err(anyhow!("Minimum supported go directive is 1.17"));
+        }
+    } else {
+        return Err(anyhow!("Error parsing go directive"));
+    }
+    Ok(())
+}
+
+pub struct GoMod;
+
+impl Parse for GoMod {
+    fn parse(&self, data: &str) -> anyhow::Result<Vec<Package>> {
+        let (_, go_mod) = go_mod::parse(data)
+            .finish()
+            .map_err(|e| anyhow!(e.to_string()))
+            .context("Failed to parse go.mod file")?;
+
+        check_go_directive(&go_mod.go)?;
+
+        Ok(go_mod.modules)
+    }
+
+    fn is_path_lockfile(&self, path: &Path) -> bool {
+        path.file_name() == Some(OsStr::new("go.mod"))
+    }
+
+    fn is_path_manifest(&self, path: &Path) -> bool {
+        path.file_name() == Some(OsStr::new("go.mod"))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use phylum_types::types::package::PackageType;
@@ -66,4 +110,88 @@ mod tests {
             assert!(pkgs.contains(&expected_pkg));
         }
     }
+
+    #[test]
+    fn parse_go_mod() {
+        let mut pkgs = GoMod.parse(include_str!("../../tests/fixtures/go.mod")).unwrap();
+        pkgs.sort();
+
+        let expected_pkgs = [
+            Package {
+                name: "../replacedmodule".into(),
+                version: PackageVersion::Path(Some("../replacedmodule".into())),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "example.com/newmodule".into(),
+                version: PackageVersion::FirstParty("v3.2.1".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "example.com/newmodule".into(),
+                version: PackageVersion::FirstParty("v3.2.2".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "example.com/newmodule".into(),
+                version: PackageVersion::FirstParty("v3.2.3".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "example.com/othermodule".into(),
+                version: PackageVersion::FirstParty("v1.2.3".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "github.com/go-chi/chi/v5".into(),
+                version: PackageVersion::FirstParty("v5.0.12".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "github.com/mattn/go-colorable".into(),
+                version: PackageVersion::FirstParty("v0.1.13".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "github.com/mattn/go-isatty".into(),
+                version: PackageVersion::FirstParty("v0.0.20".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "github.com/rs/zerolog".into(),
+                version: PackageVersion::FirstParty("v1.32.0".into()),
+                package_type: PackageType::Golang,
+            },
+            Package {
+                name: "golang.org/x/sys".into(),
+                version: PackageVersion::FirstParty("v0.12.0".into()),
+                package_type: PackageType::Golang,
+            },
+        ];
+
+        assert_eq!(expected_pkgs, *pkgs)
+    }
+
+    #[test]
+    fn parse_go_mod_unsupported() {
+        let go_mod_content = r#"
+            module cli/example
+            
+            go 1.14
+            
+            require (
+                github.com/go-chi/chi/v5 v5.0.12
+                github.com/rs/zerolog v1.32.0
+            )
+            
+            require (
+                github.com/mattn/go-colorable v0.1.13 // indirect
+                github.com/mattn/go-isatty v0.0.20 // indirect
+                golang.org/x/sys v0.12.0 // indirect
+            )
+        "#;
+
+        let error = GoMod.parse(go_mod_content).err().unwrap();
+        assert_eq!(error.to_string(), "Minimum supported go directive is 1.17")
+    }
 }
diff --git a/lockfile/src/lib.rs b/lockfile/src/lib.rs
@@ -19,7 +19,7 @@ use walkdir::WalkDir;
 pub use crate::cargo::Cargo;
 pub use crate::csharp::{CSProj, PackagesLock};
 pub use crate::cyclonedx::CycloneDX;
-pub use crate::golang::GoSum;
+pub use crate::golang::{GoMod, GoSum};
 pub use crate::java::{GradleLock, Pom};
 pub use crate::javascript::{PackageLock, Pnpm, YarnLock};
 pub use crate::parse_depfile::{parse_depfile, ParseError, ParsedLockfile};
@@ -61,6 +61,7 @@ pub enum LockfileFormat {
     #[serde(alias = "nuget")]
     Msbuild,
     NugetLock,
+    GoMod,
     Go,
     Cargo,
     Spdx,
@@ -102,6 +103,7 @@ impl LockfileFormat {
             LockfileFormat::Gradle => "gradle",
             LockfileFormat::Msbuild => "msbuild",
             LockfileFormat::NugetLock => "nugetlock",
+            LockfileFormat::GoMod => "gomod",
             LockfileFormat::Go => "go",
             LockfileFormat::Cargo => "cargo",
             LockfileFormat::Spdx => "spdx",
@@ -123,6 +125,7 @@ impl LockfileFormat {
             LockfileFormat::Gradle => &GradleLock,
             LockfileFormat::Msbuild => &CSProj,
             LockfileFormat::NugetLock => &PackagesLock,
+            LockfileFormat::GoMod => &GoMod,
             LockfileFormat::Go => &GoSum,
             LockfileFormat::Cargo => &Cargo,
             LockfileFormat::Spdx => &Spdx,
@@ -160,10 +163,11 @@ impl Iterator for LockfileFormatIter {
             8 => LockfileFormat::Gradle,
             9 => LockfileFormat::NugetLock,
             10 => LockfileFormat::Msbuild,
-            11 => LockfileFormat::Go,
-            12 => LockfileFormat::Cargo,
-            13 => LockfileFormat::Spdx,
-            14 => LockfileFormat::CycloneDX,
+            11 => LockfileFormat::GoMod,
+            12 => LockfileFormat::Go,
+            13 => LockfileFormat::Cargo,
+            14 => LockfileFormat::Spdx,
+            15 => LockfileFormat::CycloneDX,
             _ => return None,
         };
         self.0 += 1;
@@ -502,6 +506,7 @@ mod tests {
             ("requirements.txt", LockfileFormat::Pip),
             ("Pipfile.lock", LockfileFormat::Pipenv),
             ("poetry.lock", LockfileFormat::Poetry),
+            ("go.mod", LockfileFormat::GoMod),
             ("go.sum", LockfileFormat::Go),
             ("Cargo.lock", LockfileFormat::Cargo),
             (".spdx.json", LockfileFormat::Spdx),
@@ -532,6 +537,7 @@ mod tests {
             ("nuget", LockfileFormat::Msbuild),
             ("msbuild", LockfileFormat::Msbuild),
             ("nugetlock", LockfileFormat::NugetLock),
+            ("gomod", LockfileFormat::GoMod),
             ("go", LockfileFormat::Go),
             ("cargo", LockfileFormat::Cargo),
             ("spdx", LockfileFormat::Spdx),
@@ -561,6 +567,7 @@ mod tests {
             ("gradle", LockfileFormat::Gradle),
             ("msbuild", LockfileFormat::Msbuild),
             ("nugetlock", LockfileFormat::NugetLock),
+            ("gomod", LockfileFormat::GoMod),
             ("go", LockfileFormat::Go),
             ("cargo", LockfileFormat::Cargo),
             ("spdx", LockfileFormat::Spdx),
@@ -604,6 +611,7 @@ mod tests {
             (LockfileFormat::Gradle, 1),
             (LockfileFormat::Msbuild, 2),
             (LockfileFormat::NugetLock, 1),
+            (LockfileFormat::GoMod, 1),
             (LockfileFormat::Go, 1),
             (LockfileFormat::Cargo, 3),
             (LockfileFormat::Spdx, 6),