Skip to content

Commit

Permalink
Add Poetry Build (#104)
Browse files Browse the repository at this point in the history 
* Add poetry build support

* Add bi poetry command

* Handle poetry package case-insensitive names
  • Loading branch information
@talarian1
talarian1 authored Sep 28, 2022
1 parent f13e457 commit ab4690a
Show file tree
Hide file tree
Showing 5 changed files with 309 additions and 146 deletions.
12 changes: 10 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,16 @@ Note: checksums calculation is not yet supported for pip projects.
bi pipenv [pipenv command] [command options]
```

Note: checksums calculation is not yet supported for pipenv projects.

#### poetry

```shell
bi poetry [poetry command] [command options]
```

Note: checksums calculation is not yet supported for poetry projects.

#### Dotnet

```shell
Expand All @@ -404,8 +414,6 @@ bi dotnet [Dotnet command] [command options]
bi nuget [Nuget command] [command options]
```

Note: checksums calculation is not yet supported for pipenv projects.

#### Conversion to CycloneDX

You can generate build-info and have it converted into the CycloneDX format by adding to the
Expand Down
128 changes: 2 additions & 126 deletions build/python.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,9 @@ package build
import (
"fmt"
"os"
"regexp"
"strings"

"github.com/jfrog/build-info-go/entities"
"github.com/jfrog/build-info-go/utils"
"github.com/jfrog/build-info-go/utils/pythonutils"
gofrogcmd "github.com/jfrog/gofrog/io"
)

type PythonModule struct {
Expand All @@ -33,7 +29,7 @@ func newPythonModule(srcPath string, tool pythonutils.PythonTool, containingBuil
}

func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) error {
dependenciesMap, err := pm.InstallWithLogParsing(commandArgs)
dependenciesMap, err := pythonutils.GetPythonDependenciesFiles(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath)
if err != nil {
return err
}
Expand Down Expand Up @@ -71,127 +67,7 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e
// Run install command while parsing the logs for downloaded packages.
// Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...).
func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) {
log := pm.containingBuild.logger
if pm.tool == pythonutils.Pipenv {
// Add verbosity flag to pipenv commands to collect necessary data
commandArgs = append(commandArgs, "-v")
}
installCmd := utils.NewCommand(string(pm.tool), "install", commandArgs)
installCmd.Dir = pm.srcPath

dependenciesMap := map[string]entities.Dependency{}

// Create regular expressions for log parsing.
collectingRegexp, err := regexp.Compile(`^Collecting\s(\w[\w-.]+)`)
if err != nil {
return nil, err
}
downloadingRegexp, err := regexp.Compile(`^\s*Downloading\s([^\s]*)\s\(`)
if err != nil {
return nil, err
}
usingCachedRegexp, err := regexp.Compile(`^\s*Using\scached\s([\S]+)\s\(`)
if err != nil {
return nil, err
}
alreadySatisfiedRegexp, err := regexp.Compile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`)
if err != nil {
return nil, err
}

var packageName string
expectingPackageFilePath := false

// Extract downloaded package name.
dependencyNameParser := gofrogcmd.CmdOutputPattern{
RegExp: collectingRegexp,
ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
// If this pattern matched a second time before downloaded-file-name was found, prompt a message.
if expectingPackageFilePath {
// This may occur when a package-installation file is saved in pip-cache-dir, thus not being downloaded during the installation.
// Re-running pip-install with 'no-cache-dir' fixes this issue.
log.Debug(fmt.Sprintf("Could not resolve download path for package: %s, continuing...", packageName))

// Save package with empty file path.
dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: ""}
}

// Check for out of bound results.
if len(pattern.MatchedResults)-1 < 0 {
log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line))
return pattern.Line, nil
}

// Save dependency information.
expectingPackageFilePath = true
packageName = pattern.MatchedResults[1]

return pattern.Line, nil
},
}

// Extract downloaded file, stored in Artifactory.
downloadedFileParser := gofrogcmd.CmdOutputPattern{
RegExp: downloadingRegexp,
ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
// Check for out of bound results.
if len(pattern.MatchedResults)-1 < 0 {
log.Debug(fmt.Sprintf("Failed extracting download path from line: %s", pattern.Line))
return pattern.Line, nil
}

// If this pattern matched before package-name was found, do not collect this path.
if !expectingPackageFilePath {
log.Debug(fmt.Sprintf("Could not resolve package name for download path: %s , continuing...", packageName))
return pattern.Line, nil
}

// Save dependency information.
filePath := pattern.MatchedResults[1]
lastSlashIndex := strings.LastIndex(filePath, "/")
var fileName string
if lastSlashIndex == -1 {
fileName = filePath
} else {
fileName = filePath[lastSlashIndex+1:]
}
dependenciesMap[strings.ToLower(packageName)] = entities.Dependency{Id: fileName}
expectingPackageFilePath = false

log.Debug(fmt.Sprintf("Found package: %s installed with: %s", packageName, fileName))
return pattern.Line, nil
},
}

cachedFileParser := gofrogcmd.CmdOutputPattern{
RegExp: usingCachedRegexp,
ExecFunc: downloadedFileParser.ExecFunc,
}

// Extract already installed packages names.
installedPackagesParser := gofrogcmd.CmdOutputPattern{
RegExp: alreadySatisfiedRegexp,
ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
// Check for out of bound results.
if len(pattern.MatchedResults)-1 < 0 {
log.Debug(fmt.Sprintf("Failed extracting package name from line: %s", pattern.Line))
return pattern.Line, nil
}

// Save dependency with empty file name.
dependenciesMap[strings.ToLower(pattern.MatchedResults[1])] = entities.Dependency{Id: ""}
log.Debug(fmt.Sprintf("Found package: %s already installed", pattern.MatchedResults[1]))
return pattern.Line, nil
},
}

// Execute command.
var errorOut string
_, errorOut, _, err = gofrogcmd.RunCmdWithOutputParser(installCmd, true, &dependencyNameParser, &downloadedFileParser, &cachedFileParser, &installedPackagesParser)
if err != nil {
return nil, fmt.Errorf("failed running %s command with error: '%s - %s'", string(pm.tool), err.Error(), errorOut)
}
return dependenciesMap, nil
return pythonutils.InstallWithLogParsing(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath)
}

func (pm *PythonModule) SetName(name string) {
Expand Down
41 changes: 38 additions & 3 deletions cli/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@ import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"

cdx "github.com/CycloneDX/cyclonedx-go"
"github.com/jfrog/build-info-go/build"
"github.com/jfrog/build-info-go/utils"
"github.com/jfrog/build-info-go/utils/pythonutils"
"github.com/pkg/errors"
clitool "github.com/urfave/cli/v2"
"os"
"os/exec"
"strings"
)

const (
Expand Down Expand Up @@ -307,6 +308,40 @@ func GetCommands(logger utils.Log) []*clitool.Command {
}
},
},
{
Name: "poetry",
Usage: "Generate build-info for a poetry project",
UsageText: "bi pipenv",
Flags: flags,
Action: func(context *clitool.Context) (err error) {
service := build.NewBuildInfoService()
service.SetLogger(logger)
bld, err := service.GetOrCreateBuild("poetry-build", "1")
if err != nil {
return
}
defer func() {
e := bld.Clean()
if err == nil {
err = e
}
}()
pythonModule, err := bld.AddPythonModule("", pythonutils.Poetry)
if err != nil {
return
}
filteredArgs := filterCliFlags(context.Args().Slice(), flags)
if filteredArgs[0] == "install" {
err = pythonModule.RunInstallAndCollectDependencies(filteredArgs[1:])
if err != nil {
return
}
return printBuild(bld, context.String(formatFlag))
} else {
return exec.Command("poetry", filteredArgs[1:]...).Run()
}
},
},
}
}

Expand Down
Loading

0 comments on commit ab4690a

Please sign in to comment.