From ab4690a33f68d637db18fe528962297af21f52f9 Mon Sep 17 00:00:00 2001 From: Tal Arian Date: Wed, 28 Sep 2022 11:19:08 +0300 Subject: [PATCH] Add Poetry Build (#104) * Add poetry build support * Add bi poetry command * Handle poetry package case-insensitive names --- README.md | 12 ++- build/python.go | 128 +------------------------- cli/cli.go | 41 ++++++++- utils/pythonutils/poetryutils.go | 121 ++++++++++++++++++++++-- utils/pythonutils/utils.go | 153 +++++++++++++++++++++++++++++-- 5 files changed, 309 insertions(+), 146 deletions(-) diff --git a/README.md b/README.md index c80bd026..8355cd14 100644 --- a/README.md +++ b/README.md @@ -392,6 +392,16 @@ Note: checksums calculation is not yet supported for pip projects. bi pipenv [pipenv command] [command options] ``` +Note: checksums calculation is not yet supported for pipenv projects. + +#### poetry + +```shell +bi poetry [poetry command] [command options] +``` + +Note: checksums calculation is not yet supported for poetry projects. + #### Dotnet ```shell @@ -404,8 +414,6 @@ bi dotnet [Dotnet command] [command options] bi nuget [Nuget command] [command options] ``` -Note: checksums calculation is not yet supported for pipenv projects. - #### Conversion to CycloneDX You can generate build-info and have it converted into the CycloneDX format by adding to the diff --git a/build/python.go b/build/python.go index 0e46e7f2..027c29dd 100644 --- a/build/python.go +++ b/build/python.go @@ -3,13 +3,9 @@ package build import ( "fmt" "os" - "regexp" - "strings" "github.com/jfrog/build-info-go/entities" - "github.com/jfrog/build-info-go/utils" "github.com/jfrog/build-info-go/utils/pythonutils" - gofrogcmd "github.com/jfrog/gofrog/io" ) type PythonModule struct { @@ -33,7 +29,7 @@ func newPythonModule(srcPath string, tool pythonutils.PythonTool, containingBuil } func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) error { - dependenciesMap, err := pm.InstallWithLogParsing(commandArgs) + dependenciesMap, err := pythonutils.GetPythonDependenciesFiles(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath) if err != nil { return err } @@ -71,127 +67,7 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e // Run install command while parsing the logs for downloaded packages. // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...). func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) { - log := pm.containingBuild.logger - if pm.tool == pythonutils.Pipenv { - // Add verbosity flag to pipenv commands to collect necessary data - commandArgs = append(commandArgs, "-v") - } - installCmd := utils.NewCommand(string(pm.tool), "install", commandArgs) - installCmd.Dir = pm.srcPath - - dependenciesMap := map[string]entities.Dependency{} - - // Create regular expressions for log parsing. - collectingRegexp, err := regexp.Compile(`^Collecting\s(\w[\w-.]+)`) - if err != nil { - return nil, err - } - downloadingRegexp, err := regexp.Compile(`^\s*Downloading\s([^\s]*)\s\(`) - if err != nil { - return nil, err - } - usingCachedRegexp, err := regexp.Compile(`^\s*Using\scached\s([\S]+)\s\(`) - if err != nil { - return nil, err - } - alreadySatisfiedRegexp, err := regexp.Compile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`) - if err != nil { - return nil, err - } - - var packageName string - expectingPackageFilePath := false - - // Extract downloaded package name. - dependencyNameParser := gofrogcmd.CmdOutputPattern{ - RegExp: collectingRegexp, - ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { - // If this pattern matched a second time before downloaded-file-name was found, prompt a message. - if expectingPackageFilePath { - // This may occur when a package-installation file is saved in pip-cache-dir, thus not being downloaded during the installation. - // Re-running pip-install with 'no-cache-dir' fixes this issue. - log.Debug(fmt.Sprintf("Could not resolve download path for package: %s, continuing...", packageName)) - - // Save package with empty file path. - dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: ""} - } - - // Check for out of bound results. - if len(pattern.MatchedResults)-1 < 0 { - log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line)) - return pattern.Line, nil - } - - // Save dependency information. - expectingPackageFilePath = true - packageName = pattern.MatchedResults[1] - - return pattern.Line, nil - }, - } - - // Extract downloaded file, stored in Artifactory. - downloadedFileParser := gofrogcmd.CmdOutputPattern{ - RegExp: downloadingRegexp, - ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { - // Check for out of bound results. - if len(pattern.MatchedResults)-1 < 0 { - log.Debug(fmt.Sprintf("Failed extracting download path from line: %s", pattern.Line)) - return pattern.Line, nil - } - - // If this pattern matched before package-name was found, do not collect this path. - if !expectingPackageFilePath { - log.Debug(fmt.Sprintf("Could not resolve package name for download path: %s , continuing...", packageName)) - return pattern.Line, nil - } - - // Save dependency information. - filePath := pattern.MatchedResults[1] - lastSlashIndex := strings.LastIndex(filePath, "/") - var fileName string - if lastSlashIndex == -1 { - fileName = filePath - } else { - fileName = filePath[lastSlashIndex+1:] - } - dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: fileName} - expectingPackageFilePath = false - - log.Debug(fmt.Sprintf("Found package: %s installed with: %s", packageName, fileName)) - return pattern.Line, nil - }, - } - - cachedFileParser := gofrogcmd.CmdOutputPattern{ - RegExp: usingCachedRegexp, - ExecFunc: downloadedFileParser.ExecFunc, - } - - // Extract already installed packages names. - installedPackagesParser := gofrogcmd.CmdOutputPattern{ - RegExp: alreadySatisfiedRegexp, - ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { - // Check for out of bound results. - if len(pattern.MatchedResults)-1 < 0 { - log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line)) - return pattern.Line, nil - } - - // Save dependency with empty file name. - dependenciesMap[strings.ToLower(pattern.MatchedResults[1])] = entities.Dependency{Id: ""} - log.Debug(fmt.Sprintf("Found package: %s already installed", pattern.MatchedResults[1])) - return pattern.Line, nil - }, - } - - // Execute command. - var errorOut string - _, errorOut, _, err = gofrogcmd.RunCmdWithOutputParser(installCmd, true, &dependencyNameParser, &downloadedFileParser, &cachedFileParser, &installedPackagesParser) - if err != nil { - return nil, fmt.Errorf("failed running %s command with error: '%s - %s'", string(pm.tool), err.Error(), errorOut) - } - return dependenciesMap, nil + return pythonutils.InstallWithLogParsing(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath) } func (pm *PythonModule) SetName(name string) { diff --git a/cli/cli.go b/cli/cli.go index 6f28032f..6a4beef4 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -4,15 +4,16 @@ import ( "bytes" "encoding/json" "fmt" + "os" + "os/exec" + "strings" + cdx "github.com/CycloneDX/cyclonedx-go" "github.com/jfrog/build-info-go/build" "github.com/jfrog/build-info-go/utils" "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/pkg/errors" clitool "github.com/urfave/cli/v2" - "os" - "os/exec" - "strings" ) const ( @@ -307,6 +308,40 @@ func GetCommands(logger utils.Log) []*clitool.Command { } }, }, + { + Name: "poetry", + Usage: "Generate build-info for a poetry project", + UsageText: "bi pipenv", + Flags: flags, + Action: func(context *clitool.Context) (err error) { + service := build.NewBuildInfoService() + service.SetLogger(logger) + bld, err := service.GetOrCreateBuild("poetry-build", "1") + if err != nil { + return + } + defer func() { + e := bld.Clean() + if err == nil { + err = e + } + }() + pythonModule, err := bld.AddPythonModule("", pythonutils.Poetry) + if err != nil { + return + } + filteredArgs := filterCliFlags(context.Args().Slice(), flags) + if filteredArgs[0] == "install" { + err = pythonModule.RunInstallAndCollectDependencies(filteredArgs[1:]) + if err != nil { + return + } + return printBuild(bld, context.String(formatFlag)) + } else { + return exec.Command("poetry", filteredArgs[1:]...).Run() + } + }, + }, } } diff --git a/utils/pythonutils/poetryutils.go b/utils/pythonutils/poetryutils.go index ae86739b..08c9b0c9 100644 --- a/utils/pythonutils/poetryutils.go +++ b/utils/pythonutils/poetryutils.go @@ -1,10 +1,19 @@ package pythonutils import ( + "encoding/json" "errors" + "fmt" + "io/ioutil" "os" + "path/filepath" + "regexp" + "strings" "github.com/BurntSushi/toml" + "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/build-info-go/utils" + gofrogcmd "github.com/jfrog/gofrog/io" "golang.org/x/exp/maps" ) @@ -28,27 +37,27 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep filePath, err := getPoetryLockFilePath(srcPath) if err != nil || filePath == "" { // Error was returned or poetry.lock does not exist in directory. - return nil, nil, err + return map[string][]string{}, []string{}, err } projectName, directDependencies, err := getPackageNameFromPyproject(srcPath) if err != nil { - return nil, nil, err + return map[string][]string{}, []string{}, err } // Extract packages names from poetry.lock dependencies, dependenciesVersions, err := extractPackagesFromPoetryLock(filePath) if err != nil { - return nil, nil, err + return map[string][]string{}, []string{}, err } graph = make(map[string][]string) // Add the root node - the project itself. for _, directDependency := range directDependencies { - directDependencyName := directDependency + ":" + dependenciesVersions[directDependency] + directDependencyName := directDependency + ":" + dependenciesVersions[strings.ToLower(directDependency)] graph[projectName] = append(graph[projectName], directDependencyName) } // Add versions to all dependencies for dependency, transitiveDependencies := range dependencies { for _, transitiveDependency := range transitiveDependencies { - transitiveDependencyName := transitiveDependency + ":" + dependenciesVersions[transitiveDependency] + transitiveDependencyName := transitiveDependency + ":" + dependenciesVersions[strings.ToLower(transitiveDependency)] graph[dependency] = append(graph[dependency], transitiveDependencyName) } } @@ -81,7 +90,7 @@ func getPoetryLockFilePath(srcPath string) (string, error) { return getFilePath(srcPath, "poetry.lock") } -// Get the project-name by parsing the pyproject.toml file +// Get the project-name by parsing the pyproject.toml file. func extractProjectFromPyproject(pyprojectFilePath string) (project PoetryPackage, err error) { content, err := os.ReadFile(pyprojectFilePath) if err != nil { @@ -116,9 +125,107 @@ func extractPackagesFromPoetryLock(lockFilePath string) (dependencies map[string dependenciesVersions = make(map[string]string) dependencies = make(map[string][]string) for _, dependency := range poetryLockFile.Package { - dependenciesVersions[dependency.Name] = dependency.Version + dependenciesVersions[strings.ToLower(dependency.Name)] = dependency.Version dependencyName := dependency.Name + ":" + dependency.Version dependencies[dependencyName] = maps.Keys(dependency.Dependencies) } return } + +// Get the project dependencies files (.whl or .tar.gz) by searching the Python package site. +// extractPoetryDependenciesFiles returns a dictionary where the key is the dependency name and the value is a dependency file struct. +func extractPoetryDependenciesFiles(srcPath string, cmdArgs []string, log utils.Log) (dependenciesFiles map[string]entities.Dependency, err error) { + // Run poetry install and extract the site-packages location + sitePackagesPath, err := getSitePackagesPath(cmdArgs, srcPath) + if err != nil { + return + } + // Extract packages names from poetry.lock + filePath, err := getPoetryLockFilePath(srcPath) + if err != nil || filePath == "" { + // Error was returned or poetry.lock does not exist in directory. + return nil, err + } + _, dependenciesVersions, err := extractPackagesFromPoetryLock(filePath) + if err != nil { + return nil, err + } + dependenciesFiles = map[string]entities.Dependency{} + for dependency, version := range dependenciesVersions { + directUrlPath := fmt.Sprintf("%s%s-%s.dist-info%sdirect_url.json", sitePackagesPath, dependency, version, string(os.PathSeparator)) + directUrlFile, err := ioutil.ReadFile(directUrlPath) + if err != nil { + log.Debug(fmt.Sprintf("Could not resolve download path for package: %s, error: %s \ncontinuing...", dependency, err)) + continue + } + directUrl := packagedDirectUrl{} + err = json.Unmarshal(directUrlFile, &directUrl) + if err != nil { + log.Debug(fmt.Sprintf("Could not resolve download path for package: %s, error: %s \ncontinuing...", dependency, err)) + continue + } + lastSeparatorIndex := strings.LastIndex(directUrl.Url, string(os.PathSeparator)) + var fileName string + if lastSeparatorIndex == -1 { + fileName = directUrl.Url + } else { + fileName = directUrl.Url[lastSeparatorIndex+1:] + } + dependenciesFiles[strings.ToLower(dependency)] = entities.Dependency{Id: fileName} + log.Debug(fmt.Sprintf("Found package: %s installed with: %s", dependency, fileName)) + + } + return +} + +func getSitePackagesPath(commandArgs []string, srcPath string) (sitePackagesPath string, err error) { + // First run poetry install with verbose logging + commandArgs = append(commandArgs, "-vv") + installCmd := utils.NewCommand("poetry", "install", commandArgs) + installCmd.Dir = srcPath + // Extract the virtuL env path + virtualEnvRegexp, err := regexp.Compile(`^Using\svirtualenv:\s(.*)$`) + if err != nil { + return "", err + } + virtualEnvNameParser := gofrogcmd.CmdOutputPattern{ + RegExp: virtualEnvRegexp, + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + + // Check for out of bound results. + if len(pattern.MatchedResults)-1 < 0 { + return "", nil + } + // If found, return the virtual env path + return pattern.MatchedResults[1], nil + }, + } + virtualEnvPath, errorOut, _, err := gofrogcmd.RunCmdWithOutputParser(installCmd, true, &virtualEnvNameParser) + if err != nil { + return "", fmt.Errorf("failed running poetry command with error: '%s - %s'", err.Error(), errorOut) + } + if virtualEnvPath != "" { + // Take the first line matches the virtualEnvRegexp + sitePackagesPath = strings.Split(virtualEnvPath, "\n")[0] + // Extract from poetry env(i.e PROJECT-9SrbZw5z-py3.9) the env python version + pythonVersionIndex := strings.LastIndex(sitePackagesPath, "-py") + if pythonVersionIndex == -1 { + return "", fmt.Errorf("failed extracting python site package form the following virtual env %q", sitePackagesPath) + } + pythonVersion := sitePackagesPath[pythonVersionIndex+3:] + // add /lib/python3.10/site-packages + sitePackagesPath = filepath.Join(sitePackagesPath, "lib", "python"+pythonVersion, "site-packages") + string(os.PathSeparator) + } else { + // If no virtuL env is use, return the local python installation site-packages path + siteCmd := utils.NewCommand("python", "site", []string{"-m", "--user-site"}) + sitePackagesPath, err = gofrogcmd.RunCmdOutput(siteCmd) + if err != nil { + return "", fmt.Errorf("failed running python -m site --user-site with error: '%s'", err.Error()) + } + } + return +} + +type packagedDirectUrl struct { + Url string +} diff --git a/utils/pythonutils/utils.go b/utils/pythonutils/utils.go index 08778667..5e83e001 100644 --- a/utils/pythonutils/utils.go +++ b/utils/pythonutils/utils.go @@ -2,17 +2,20 @@ package pythonutils import ( "errors" + "fmt" "path/filepath" + "regexp" "strings" - buildinfo "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/build-info-go/entities" "github.com/jfrog/build-info-go/utils" + gofrogcmd "github.com/jfrog/gofrog/io" ) const ( - Pip = "pip" - Pipenv = "pipenv" - Poetry = "poetry" + Pip PythonTool = "pip" + Pipenv PythonTool = "pipenv" + Poetry PythonTool = "poetry" ) type PythonTool string @@ -54,6 +57,17 @@ type packageType struct { InstalledVersion string `json:"installed_version,omitempty"` } +func GetPythonDependenciesFiles(tool PythonTool, args []string, log utils.Log, srcPath string) (map[string]entities.Dependency, error) { + switch tool { + case Pip, Pipenv: + return InstallWithLogParsing(tool, args, log, srcPath) + case Poetry: + return extractPoetryDependenciesFiles(srcPath, args, log) + default: + return nil, errors.New(string(tool) + " commands are not supported.") + } +} + func GetPythonDependencies(tool PythonTool, srcPath, localDependenciesPath string) (dependenciesGraph map[string][]string, topLevelDependencies []string, err error) { switch tool { case Pip: @@ -86,7 +100,7 @@ func GetPackageName(tool PythonTool, srcPath string) (packageName string, err er // topLevelPackagesList - The direct dependencies // packageName - The resolved package name of the Python project, may be empty if we couldn't resolve it // moduleName - The input module name from the user, or the packageName -func UpdateDepsIdsAndRequestedBy(dependenciesMap map[string]buildinfo.Dependency, dependenciesGraph map[string][]string, topLevelPackagesList []string, packageName, moduleName string) { +func UpdateDepsIdsAndRequestedBy(dependenciesMap map[string]entities.Dependency, dependenciesGraph map[string][]string, topLevelPackagesList []string, packageName, moduleName string) { if packageName == "" { // Projects without setup.py dependenciesGraph[moduleName] = topLevelPackagesList @@ -94,15 +108,15 @@ func UpdateDepsIdsAndRequestedBy(dependenciesMap map[string]buildinfo.Dependency // Projects with setup.py dependenciesGraph[moduleName] = dependenciesGraph[packageName] } - rootModule := buildinfo.Dependency{Id: moduleName, RequestedBy: [][]string{{}}} + rootModule := entities.Dependency{Id: moduleName, RequestedBy: [][]string{{}}} updateDepsIdsAndRequestedBy(rootModule, dependenciesMap, dependenciesGraph) } -func updateDepsIdsAndRequestedBy(parentDependency buildinfo.Dependency, dependenciesMap map[string]buildinfo.Dependency, dependenciesGraph map[string][]string) { +func updateDepsIdsAndRequestedBy(parentDependency entities.Dependency, dependenciesMap map[string]entities.Dependency, dependenciesGraph map[string][]string) { for _, childId := range dependenciesGraph[parentDependency.Id] { childName := childId[0:strings.Index(childId, ":")] if childDep, ok := dependenciesMap[childName]; ok { - if childDep.NodeHasLoop() || len(childDep.RequestedBy) >= buildinfo.RequestedByMaxLength { + if childDep.NodeHasLoop() || len(childDep.RequestedBy) >= entities.RequestedByMaxLength { continue } // Update RequestedBy field from parent's RequestedBy. @@ -137,3 +151,126 @@ func getFilePath(srcPath, fileName string) (string, error) { } return filePath, nil } + +func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, srcPath string) (map[string]entities.Dependency, error) { + if tool == Pipenv { + // Add verbosity flag to pipenv commands to collect necessary data + commandArgs = append(commandArgs, "-v") + } + installCmd := utils.NewCommand(string(tool), "install", commandArgs) + installCmd.Dir = srcPath + + dependenciesMap := map[string]entities.Dependency{} + + // Create regular expressions for log parsing. + collectingRegexp, err := regexp.Compile(`^Collecting\s(\w[\w-.]+)`) + if err != nil { + return nil, err + } + downloadingRegexp, err := regexp.Compile(`^\s*Downloading\s([^\s]*)\s\(`) + if err != nil { + return nil, err + } + usingCachedRegexp, err := regexp.Compile(`^\s*Using\scached\s([\S]+)\s\(`) + if err != nil { + return nil, err + } + alreadySatisfiedRegexp, err := regexp.Compile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`) + if err != nil { + return nil, err + } + + var packageName string + expectingPackageFilePath := false + + // Extract downloaded package name. + dependencyNameParser := gofrogcmd.CmdOutputPattern{ + RegExp: collectingRegexp, + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + // If this pattern matched a second time before downloaded-file-name was found, prompt a message. + if expectingPackageFilePath { + // This may occur when a package-installation file is saved in pip-cache-dir, thus not being downloaded during the installation. + // Re-running pip-install with 'no-cache-dir' fixes this issue. + log.Debug(fmt.Sprintf("Could not resolve download path for package: %s, continuing...", packageName)) + + // Save package with empty file path. + dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: ""} + } + + // Check for out of bound results. + if len(pattern.MatchedResults)-1 < 0 { + log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line)) + return pattern.Line, nil + } + + // Save dependency information. + expectingPackageFilePath = true + packageName = pattern.MatchedResults[1] + + return pattern.Line, nil + }, + } + + // Extract downloaded file, stored in Artifactory. + downloadedFileParser := gofrogcmd.CmdOutputPattern{ + RegExp: downloadingRegexp, + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + // Check for out of bound results. + if len(pattern.MatchedResults)-1 < 0 { + log.Debug(fmt.Sprintf("Failed extracting download path from line: %s", pattern.Line)) + return pattern.Line, nil + } + + // If this pattern matched before package-name was found, do not collect this path. + if !expectingPackageFilePath { + log.Debug(fmt.Sprintf("Could not resolve package name for download path: %s , continuing...", packageName)) + return pattern.Line, nil + } + + // Save dependency information. + filePath := pattern.MatchedResults[1] + lastSlashIndex := strings.LastIndex(filePath, "/") + var fileName string + if lastSlashIndex == -1 { + fileName = filePath + } else { + fileName = filePath[lastSlashIndex+1:] + } + dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: fileName} + expectingPackageFilePath = false + + log.Debug(fmt.Sprintf("Found package: %s installed with: %s", packageName, fileName)) + return pattern.Line, nil + }, + } + + cachedFileParser := gofrogcmd.CmdOutputPattern{ + RegExp: usingCachedRegexp, + ExecFunc: downloadedFileParser.ExecFunc, + } + + // Extract already installed packages names. + installedPackagesParser := gofrogcmd.CmdOutputPattern{ + RegExp: alreadySatisfiedRegexp, + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + // Check for out of bound results. + if len(pattern.MatchedResults)-1 < 0 { + log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line)) + return pattern.Line, nil + } + + // Save dependency with empty file name. + dependenciesMap[strings.ToLower(pattern.MatchedResults[1])] = entities.Dependency{Id: ""} + log.Debug(fmt.Sprintf("Found package: %s already installed", pattern.MatchedResults[1])) + return pattern.Line, nil + }, + } + + // Execute command. + var errorOut string + _, errorOut, _, err = gofrogcmd.RunCmdWithOutputParser(installCmd, true, &dependencyNameParser, &downloadedFileParser, &cachedFileParser, &installedPackagesParser) + if err != nil { + return nil, fmt.Errorf("failed running %s command with error: '%s - %s'", string(tool), err.Error(), errorOut) + } + return dependenciesMap, nil +}