From 7333ca47ab9e2e6191b150d6b3f7404fc20f5082 Mon Sep 17 00:00:00 2001 From: Sara Omari <114062096+sarao1310@users.noreply.github.com> Date: Tue, 14 Mar 2023 21:17:05 +0200 Subject: [PATCH 1/3] Enhancements to Maven Home (#133) --- build/maven.go | 136 +++++++++++++++++++++++++++++++------------- build/maven_test.go | 60 +++++++++++++++++++ 2 files changed, 157 insertions(+), 39 deletions(-) diff --git a/build/maven.go b/build/maven.go index d96e8bf5..48a025e4 100644 --- a/build/maven.go +++ b/build/maven.go @@ -4,13 +4,11 @@ import ( "bytes" "errors" "fmt" + "github.com/jfrog/build-info-go/utils" "os" "os/exec" "path/filepath" - "runtime" "strings" - - "github.com/jfrog/build-info-go/utils" ) const ( @@ -55,6 +53,8 @@ type extractorDetails struct { props map[string]string // Local path to the configuration file. propsDir string + // Use the maven wrapper to build the project. + useWrapper bool } // Add a new Maven module to a given build. @@ -81,11 +81,12 @@ func newMavenModule(containingBuild *Build, srcPath string) (*MavenModule, error }, err } -func (mm *MavenModule) SetExtractorDetails(localdExtractorPath, extractorPropsdir string, goals []string, downloadExtractorFunc func(downloadTo, downloadFrom string) error, configProps map[string]string) *MavenModule { - mm.extractorDetails.localPath = localdExtractorPath +func (mm *MavenModule) SetExtractorDetails(localExtractorPath, extractorPropsdir string, goals []string, downloadExtractorFunc func(downloadTo, downloadFrom string) error, configProps map[string]string, useWrapper bool) *MavenModule { + mm.extractorDetails.localPath = localExtractorPath mm.extractorDetails.propsDir = extractorPropsdir mm.extractorDetails.downloadExtractorFunc = downloadExtractorFunc mm.extractorDetails.goals = goals + mm.extractorDetails.useWrapper = useWrapper if configProps != nil { mm.extractorDetails.props = configProps } @@ -144,24 +145,30 @@ func (mm *MavenModule) createMvnRunConfig() (*mvnRunConfig, error) { } // Generates Maven build-info. -func (mm *MavenModule) CalcDependencies() error { +func (mm *MavenModule) CalcDependencies() (err error) { if mm.srcPath == "" { - var err error if mm.srcPath, err = os.Getwd(); err != nil { return err } } - err := downloadMavenExtractor(mm.extractorDetails.localPath, mm.extractorDetails.downloadExtractorFunc, mm.containingBuild.logger) - if err != nil { - return err + if err = downloadMavenExtractor(mm.extractorDetails.localPath, mm.extractorDetails.downloadExtractorFunc, mm.containingBuild.logger); err != nil { + return } mvnRunConfig, err := mm.createMvnRunConfig() if err != nil { - return err + return } + defer func() { + fileExist, e := utils.IsFileExists(mvnRunConfig.buildInfoProperties, false) + if fileExist && e == nil { + e = os.Remove(mvnRunConfig.buildInfoProperties) + } + if err == nil { + err = e + } + }() - defer os.Remove(mvnRunConfig.buildInfoProperties) mm.containingBuild.logger.Info("Running Mvn...") return mvnRunConfig.runCmd() } @@ -172,42 +179,93 @@ func (mm *MavenModule) loadMavenHome() (mavenHome string, err error) { if mavenHome == "" { // The 'mavenHome' is not defined. // Since Maven installation can be located in different locations, - // Depending on the installation type and the OS (for example: For Mac with brew install: /usr/local/Cellar/maven/{version}/libexec or Ubuntu with debian: /usr/share/maven), - // We need to grab the location using the mvn --version command - + // depending on the installation type and the OS (for example: For Mac with brew install: /usr/local/Cellar/maven/{version}/libexec or Ubuntu with debian: /usr/share/maven), + // we need to grab the location using the mvn --version command. // First we will try lo look for 'mvn' in PATH. - mvnPath, err := exec.LookPath("mvn") - if err != nil || mvnPath == "" { - return "", errors.New(err.Error() + "Hint: The mvn command may not be included in the PATH. Either add it to the path, or set the M2_HOME environment variable value to the maven installation directory, which is the directory which includes the bin and lib directories.") + maven, err := mm.getExecutableName() + if err != nil { + return maven, err } - mm.containingBuild.logger.Debug(MavenHome, " is not defined. Retrieving Maven home using 'mvn --version' command.") - cmd := exec.Command("mvn", "--version") - var stdout bytes.Buffer - cmd.Stdout = &stdout - err = cmd.Run() + if !mm.extractorDetails.useWrapper { + mvnPath, err := mm.lookPath() + if err != nil { + return mvnPath, err + } + } + versionOutput, err := mm.execMavenVersion(maven) if err != nil { return "", err } - output := strings.Split(strings.TrimSpace(stdout.String()), "\n") // Finding the relevant "Maven home" line in command response. - for _, line := range output { - if strings.HasPrefix(line, "Maven home:") { - mavenHome = strings.Split(line, " ")[2] - if runtime.GOOS == "windows" { - mavenHome = strings.TrimSuffix(mavenHome, "\r") - } - mavenHome, err = filepath.Abs(mavenHome) - if err != nil { - return "", err - } - break - } - } - if mavenHome == "" { - return "", errors.New("Could not find the location of the maven home directory, by running 'mvn --version' command. The command output is:\n" + stdout.String() + "\nYou also have the option of setting the M2_HOME environment variable value to the maven installation directory, which is the directory which includes the bin and lib directories.") + mavenHome, err = mm.extractMavenPath(versionOutput) + if err != nil { + return "", err } } mm.containingBuild.logger.Debug("Maven home location: ", mavenHome) + + return +} + +func (mm *MavenModule) lookPath() (mvnPath string, err error) { + mvnPath, err = exec.LookPath("mvn") + err = mm.determineError(mvnPath, "", err) + return +} + +// This function generates an error with a clear message, based on the arguments it gets. +func (mm *MavenModule) determineError(mvnPath, versionOutput string, err error) error { + if err != nil { + if versionOutput == "" { + return errors.New(err.Error() + "\nHint: The mvn command may not be included in the PATH. Either add it to the path or set the M2_HOME environment variable value to the maven installation directory, which is the directory that includes the bin and lib directories.") + } + return errors.New(err.Error() + "Could not find the location of the maven home directory, by running 'mvn --version' command. The command versionOutput is:\n" + versionOutput + "\nYou also have the option of setting the M2_HOME environment variable value to the maven installation directory, which is the directory which includes the bin and lib directories.") + } + if mvnPath == "" { + if versionOutput == "" { + return errors.New("hint: The mvn command may not be included in the PATH. Either add it to the path or set the M2_HOME environment variable value to the maven installation directory, which is the directory that includes the bin and lib directories") + } + return errors.New("Could not find the location of the maven home directory, by running 'mvn --version' command. The command versionOutput is:\n" + versionOutput + "\nYou also have the option of setting the M2_HOME environment variable value to the maven installation directory, which is the directory which includes the bin and lib directories.") + } + return nil +} + +func (mm *MavenModule) getExecutableName() (maven string, err error) { + maven = "mvn" + if mm.extractorDetails.useWrapper { + if utils.IsWindows() { + maven, err = filepath.Abs("mvnw.cmd") + } else { + maven = "./mvnw" + } + } + return +} + +func (mm *MavenModule) execMavenVersion(maven string) (stdout bytes.Buffer, err error) { + mm.containingBuild.logger.Debug(MavenHome, " is not defined. Retrieving Maven home using 'mvn --version' command.") + cmd := exec.Command(maven, "--version") + cmd.Stdout = &stdout + err = cmd.Run() + err = mm.determineError("mvn", stdout.String(), err) + if err != nil { + return stdout, err + } + + return stdout, nil +} + +func (mm *MavenModule) extractMavenPath(mavenVersionOutput bytes.Buffer) (mavenHome string, err error) { + mavenVersionResultInArray := strings.Split(strings.TrimSpace(mavenVersionOutput.String()), "\n") + for _, line := range mavenVersionResultInArray { + if !strings.HasPrefix(line, "Maven home:") { + continue + } + mavenHome = strings.Split(line, " ")[2] + mavenHome = strings.TrimSpace(mavenHome) + mavenHome, err = filepath.Abs(mavenHome) + err = mm.determineError(mavenHome, mavenVersionOutput.String(), err) + } return } diff --git a/build/maven_test.go b/build/maven_test.go index 455dc4f8..1213ee4c 100644 --- a/build/maven_test.go +++ b/build/maven_test.go @@ -1,6 +1,7 @@ package build import ( + "bytes" "encoding/json" "fmt" "os" @@ -40,6 +41,7 @@ func TestGenerateBuildInfoForMavenProject(t *testing.T) { defer func() { assert.NoError(t, mavenBuild.Clean()) }() + testdataDir, err := filepath.Abs(filepath.Join("testdata")) assert.NoError(t, err) // Create maven project @@ -72,3 +74,61 @@ func getExpectedMavenBuildInfo(t *testing.T, filePath string) entities.BuildInfo assert.NoError(t, json.Unmarshal(data, &buildinfo)) return buildinfo } + +func TestExtractMavenPath(t *testing.T) { + service := NewBuildInfoService() + mavenBuild, err := service.GetOrCreateBuild("build-info-maven-test", "1") + assert.NoError(t, err) + defer assert.NoError(t, mavenBuild.Clean()) + + testdataDir, err := filepath.Abs(filepath.Join("testdata")) + assert.NoError(t, err) + // Create maven project + projectPath := filepath.Join(testdataDir, "maven", "project") + tmpProjectPath, cleanup := testdatautils.CreateTestProject(t, projectPath) + defer cleanup() + // Add maven project as module in build-info. + mavenModule, err := mavenBuild.AddMavenModule(tmpProjectPath) + assert.NoError(t, err) + + allTests := []struct { + mavenVersionResultFirstLine string + mavenVersionResultSecondLine string + mavenVersionResultThirdLine string + }{ + {"Maven home: /test/is/good", "Home: /test/is/not/good", "Mvn Home:= /test/is/not/good"}, + {"Home: /test/is/not/good", "Maven home: /test/is/good", "Mvn Home:= /test/is/not/good"}, + {"Mvn Home:= /test/is/not/good", "Home: /test/is/not/good", "Maven home: /test/is/good"}, + } + + for _, test := range allTests { + var mavenVersionFullResult []string + mavenVersionFullResult = append(mavenVersionFullResult, test.mavenVersionResultFirstLine, test.mavenVersionResultSecondLine, test.mavenVersionResultThirdLine) + data1 := bytes.Buffer{} + for _, i := range mavenVersionFullResult { + data1.WriteString(i) + data1.WriteString("\n") + } + mavenHome, err := mavenModule.extractMavenPath(data1) + assert.NoError(t, err) + if utils.IsWindows() { + assert.Equal(t, "D:\\test\\is\\good", mavenHome) + } else { + assert.Equal(t, "/test/is/good", mavenHome) + } + } +} + +func TestGetExecutableName(t *testing.T) { + // Add maven project as module in build-info. + mavenModule := MavenModule{extractorDetails: &extractorDetails{useWrapper: true}} + mvnHome, err := mavenModule.getExecutableName() + assert.NoError(t, err) + if !utils.IsWindows() { + assert.Equal(t, "./mvnw", mvnHome) + } else { + result, err := filepath.Abs("mvnw.cmd") + assert.NoError(t, err) + assert.Equal(t, result, mvnHome) + } +} From ff371731ef0525712db8cdcf83724f42c3c96283 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 15 Mar 2023 19:22:41 +0200 Subject: [PATCH 2/3] Remove redundant npm version log (#138) --- build/testdata/test_helpers.go | 2 +- build/utils/npm.go | 11 +++++------ utils/pythonutils/poetryutils.go | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/build/testdata/test_helpers.go b/build/testdata/test_helpers.go index 0b535ac7..b81b39dd 100644 --- a/build/testdata/test_helpers.go +++ b/build/testdata/test_helpers.go @@ -71,5 +71,5 @@ func PrintBuildInfoMismatch(t *testing.T, expected, actual []entities.Module) { assert.NoError(t, err) actualStr, err := json.MarshalIndent(actual, "", " ") assert.NoError(t, err) - t.Errorf("build-info don't match. want: \n %v\n got:\n%s\n", string(excpectedStr), string(actualStr)) + t.Errorf("build-info don't match. want: \n%v\ngot:\n%s\n", string(excpectedStr), string(actualStr)) } diff --git a/build/utils/npm.go b/build/utils/npm.go index 691d234b..297417d7 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -47,7 +47,7 @@ func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmA continue } if calculateChecksums { - dep.Md5, dep.Sha1, dep.Sha256, err = calculateChecksum(cacache, dep.Name, dep.Version, dep.Integrity, log) + dep.Md5, dep.Sha1, dep.Sha256, err = calculateChecksum(cacache, dep.Name, dep.Version, dep.Integrity) if err != nil { if dep.Optional { missingOptionalDeps = append(missingOptionalDeps, dep.Id) @@ -220,7 +220,7 @@ func parseDependencies(data []byte, pathToRoot []string, dependencies map[string } return errors.New("failed to parse '" + string(value) + "' from npm ls output.") } - appendDependency(dependencies, npmLsDependency, pathToRoot, log) + appendDependency(dependencies, npmLsDependency, pathToRoot) transitive, _, _, err := jsonparser.Get(value, "dependencies") if err != nil && err.Error() != "Key path not found" { return err @@ -256,7 +256,7 @@ func npmLsDependencyParser(data []byte) (*npmLsDependency, error) { return npmLsDependency, json.Unmarshal(data, &npmLsDependency) } -func appendDependency(dependencies map[string]*dependencyInfo, dep *npmLsDependency, pathToRoot []string, log utils.Log) { +func appendDependency(dependencies map[string]*dependencyInfo, dep *npmLsDependency, pathToRoot []string) { depId := dep.id() scopes := dep.getScopes() if dependencies[depId] == nil { @@ -275,7 +275,7 @@ func appendDependency(dependencies map[string]*dependencyInfo, dep *npmLsDepende } // Lookup for a dependency's tarball in npm cache, and calculate checksum. -func calculateChecksum(cacache *cacache, name, version, integrity string, log utils.Log) (md5 string, sha1 string, sha256 string, err error) { +func calculateChecksum(cacache *cacache, name, version, integrity string) (md5 string, sha1 string, sha256 string, err error) { if integrity == "" { var info *cacacheInfo info, err = cacache.GetInfo(name + "@" + version) @@ -370,7 +370,6 @@ func GetNpmVersionAndExecPath(log utils.Log) (*version.Version, string, error) { if err != nil { return nil, "", err } - log.Debug("Using npm version:", string(versionData)) return version.NewVersion(string(versionData)), npmExecPath, nil } @@ -462,5 +461,5 @@ func GetNpmConfigCache(srcPath, executablePath string, npmArgs []string, log uti } func printMissingDependenciesWarning(dependencyType string, dependencies []string, log utils.Log) { - log.Debug("The following dependencies will not be included in the build-info, because the 'npm ls' command did not return their integrity.\nThe reason why the version wasn't returned may be because the package is a '" + dependencyType + "', which was not manually installed.\n It is therefore okay to skip this dependency: " + strings.Join(dependencies, ",")) + log.Debug("The following dependencies will not be included in the build-info, because the 'npm ls' command did not return their integrity.\nThe reason why the version wasn't returned may be because the package is a '" + dependencyType + "', which was not manually installed.\nIt is therefore okay to skip this dependency: " + strings.Join(dependencies, ",")) } diff --git a/utils/pythonutils/poetryutils.go b/utils/pythonutils/poetryutils.go index 76ef8b39..f45ea484 100644 --- a/utils/pythonutils/poetryutils.go +++ b/utils/pythonutils/poetryutils.go @@ -96,7 +96,6 @@ func extractProjectFromPyproject(pyprojectFilePath string) (project PoetryPackag return } var pyprojectFile PyprojectToml - _, err = toml.Decode(string(content), &pyprojectFile) if err != nil { return From a9f6b73206d7d4f3111ec6fe05d5fbad0a54d99a Mon Sep 17 00:00:00 2001 From: Tal Arian Date: Thu, 16 Mar 2023 11:54:17 +0200 Subject: [PATCH 3/3] Upgrade Extractors version * Upgrade Maven Extractor Version to 2.39.7 * Upgrade gradle Extractor Version to 4.31.7 --- build/gradle.go | 2 +- build/maven.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/gradle.go b/build/gradle.go index 2aec6908..36ee310d 100644 --- a/build/gradle.go +++ b/build/gradle.go @@ -16,7 +16,7 @@ const ( GradleExtractorFileName = "build-info-extractor-gradle-%s-uber.jar" gradleInitScriptTemplate = "gradle.init" GradleExtractorRemotePath = "org/jfrog/buildinfo/build-info-extractor-gradle/%s" - GradleExtractorDependencyVersion = "4.31.5" + GradleExtractorDependencyVersion = "4.31.7" ) type GradleModule struct { diff --git a/build/maven.go b/build/maven.go index 48a025e4..f74641fd 100644 --- a/build/maven.go +++ b/build/maven.go @@ -4,11 +4,12 @@ import ( "bytes" "errors" "fmt" - "github.com/jfrog/build-info-go/utils" "os" "os/exec" "path/filepath" "strings" + + "github.com/jfrog/build-info-go/utils" ) const ( @@ -18,7 +19,7 @@ const ( PropertiesTempfolderName = "properties" MavenExtractorRemotePath = "org/jfrog/buildinfo/build-info-extractor-maven3/%s" GeneratedBuildInfoTempPrefix = "generatedBuildInfo" - MavenExtractorDependencyVersion = "2.39.5" + MavenExtractorDependencyVersion = "2.39.7" ClassworldsConf = `main is org.apache.maven.cli.MavenCli from plexus.core