Add twine support (#276)

README.md

@@ -395,6 +395,12 @@ bi pipenv [pipenv command] [command options]
 
 Note: checksums calculation is not yet supported for pipenv projects.
 
+#### twine
+
+```shell
+bi twine [twine command] [command options]
+```
+
 #### Dotnet
 
 ```shell

build/build.go

@@ -373,6 +373,14 @@ func (b *Build) GetBuildTimestamp() time.Time {
 	return b.buildTimestamp
 }
 
+func (b *Build) AddArtifacts(moduleId string, moduleType entities.ModuleType, artifacts ...entities.Artifact) error {
+	if !b.buildNameAndNumberProvided() {
+		return errors.New("a build name must be provided in order to add artifacts")
+	}
+	partial := &entities.Partial{ModuleId: moduleId, ModuleType: moduleType, Artifacts: artifacts}
+	return b.SavePartialBuildInfo(partial)
+}
+
 type partialModule struct {
 	moduleType   entities.ModuleType
 	artifacts    map[string]entities.Artifact

build/golang.go

@@ -55,11 +55,7 @@ func (gm *GoModule) SetName(name string) {
 }
 
 func (gm *GoModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !gm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: gm.name, ModuleType: entities.Go, Artifacts: artifacts}
-	return gm.containingBuild.SavePartialBuildInfo(partial)
+	return gm.containingBuild.AddArtifacts(gm.name, entities.Go, artifacts...)
 }
 
 func (gm *GoModule) loadDependencies() ([]entities.Dependency, error) {

build/npm.go

@@ -97,11 +97,7 @@ func (nm *NpmModule) SetCollectBuildInfo(collectBuildInfo bool) {
 }
 
 func (nm *NpmModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !nm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: nm.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return nm.containingBuild.SavePartialBuildInfo(partial)
+	return nm.containingBuild.AddArtifacts(nm.name, entities.Npm, artifacts...)
 }
 
 // This function discards the npm command in npmArgs and keeps only the command flags.

build/python.go

@@ -11,7 +11,7 @@ import (
 type PythonModule struct {
 	containingBuild            *Build
 	tool                       pythonutils.PythonTool
-	name                       string
+	id                         string
 	srcPath                    string
 	localDependenciesPath      string
 	updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error
@@ -37,41 +37,48 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e
 	if err != nil {
 		return fmt.Errorf("failed while attempting to get %s dependencies graph: %s", pm.tool, err.Error())
 	}
-	// Get package-name.
-	packageName, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
-	if pkgNameErr != nil {
-		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
-	}
-	// If module-name was set by the command, don't change it.
-	if pm.name == "" {
-		// If the package name is unknown, set the module name to be the build name.
-		pm.name = packageName
-		if pm.name == "" {
-			pm.name = pm.containingBuild.buildName
-			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.name))
-		}
-	}
+
+	packageId := pm.SetModuleId()
+
 	if pm.updateDepsChecksumInfoFunc != nil {
 		err = pm.updateDepsChecksumInfoFunc(dependenciesMap, pm.srcPath)
 		if err != nil {
 			return err
 		}
 	}
-	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageName, pm.name)
-	buildInfoModule := entities.Module{Id: pm.name, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
+	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageId, pm.id)
+	buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
 	buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}}
 
 	return pm.containingBuild.SaveBuildInfo(buildInfo)
 }
 
+// Sets the module ID and returns the package ID (if found).
+func (pm *PythonModule) SetModuleId() (packageId string) {
+	packageId, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
+	if pkgNameErr != nil {
+		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
+	}
+	// If module-name was set by the command, don't change it.
+	if pm.id == "" {
+		// If the package name is unknown, set the module name to be the build name.
+		pm.id = packageId
+		if pm.id == "" {
+			pm.id = pm.containingBuild.buildName
+			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.id))
+		}
+	}
+	return
+}
+
 // Run install command while parsing the logs for downloaded packages.
 // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...).
 func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) {
 	return pythonutils.InstallWithLogParsing(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath)
 }
 
 func (pm *PythonModule) SetName(name string) {
-	pm.name = name
+	pm.id = name
 }
 
 func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
@@ -81,3 +88,27 @@ func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
 func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error) {
 	pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc
 }
+
+func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]entities.Artifact, error) {
+	pm.SetModuleId()
+	artifactsPaths, err := pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath)
+	if err != nil {
+		return nil, err
+	}
+	return pythonutils.CreateArtifactsFromPaths(artifactsPaths)
+}
+
+func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error {
+	return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...)
+}
+
+func (pm *PythonModule) TwineUploadAndGenerateBuild(commandArgs []string) error {
+	artifacts, err := pm.TwineUploadWithLogParsing(commandArgs)
+	if err != nil {
+		return err
+	}
+
+	buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Artifacts: artifacts}
+	buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}}
+	return pm.containingBuild.SaveBuildInfo(buildInfo)
+}

build/yarn.go

@@ -149,11 +149,7 @@ func (ym *YarnModule) SetTraverseDependenciesFunc(traverseDependenciesFunc func(
 }
 
 func (ym *YarnModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !ym.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: ym.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return ym.containingBuild.SavePartialBuildInfo(partial)
+	return ym.containingBuild.AddArtifacts(ym.name, entities.Npm, artifacts...)
 }
 
 func validateYarnVersion(executablePath, srcPath string) error {

cli/cli.go

@@ -285,6 +285,36 @@ func GetCommands(logger utils.Log) []*clitool.Command {
 				}
 			},
 		},
+		{
+			Name:      "twine",
+			Usage:     "Generate build-info for a twine project",
+			UsageText: "bi twine",
+			Flags:     flags,
+			Action: func(context *clitool.Context) (err error) {
+				service := build.NewBuildInfoService()
+				service.SetLogger(logger)
+				bld, err := service.GetOrCreateBuild("twine-build", "1")
+				if err != nil {
+					return
+				}
+				defer func() {
+					err = errors.Join(err, bld.Clean())
+				}()
+				pythonModule, err := bld.AddPythonModule("", pythonutils.Twine)
+				if err != nil {
+					return
+				}
+				filteredArgs := filterCliFlags(context.Args().Slice(), flags)
+				if filteredArgs[0] == "upload" {
+					if err := pythonModule.TwineUploadAndGenerateBuild(filteredArgs[1:]); err != nil {
+						return err
+					}
+					return printBuild(bld, context.String(formatFlag))
+				} else {
+					return exec.Command("twine", filteredArgs[1:]...).Run()
+				}
+			},
+		},
 	}
 }
 

utils/pythonutils/piputils.go

@@ -73,20 +73,20 @@ func writeScriptIfNeeded(targetDirPath, scriptName string) error {
 	return nil
 }
 
-func getPackageNameFromSetuppy(srcPath string) (string, error) {
+func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVersion string, err error) {
 	filePath, err := getSetupPyFilePath(srcPath)
 	if err != nil || filePath == "" {
 		// Error was returned or setup.py does not exist in directory.
-		return "", err
+		return
 	}
 
 	// Extract package name from setup.py.
-	packageName, err := ExtractPackageNameFromSetupPy(filePath)
+	packageName, packageVersion, err = extractPackageNameFromSetupPy(filePath)
 	if err != nil {
 		// If setup.py egg_info command failed we use build name as module name and continue to pip-install execution
-		return "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
+		return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
 	}
-	return packageName, nil
+	return packageName, packageVersion, nil
 }
 
 // Look for 'setup.py' file in current work dir.
@@ -95,16 +95,16 @@ func getSetupPyFilePath(srcPath string) (string, error) {
 	return getFilePath(srcPath, "setup.py")
 }
 
-// Get the project-name by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
-func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, error) {
+// Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
+func extractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) {
 	// Execute egg_info command and return PKG-INFO content.
 	content, err := getEgginfoPkginfoContent(setuppyFilePath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	// Extract project name from file content.
-	return getProjectIdFromFileContent(content)
+	return getProjectNameAndVersionFromFileContent(content)
 }
 
 // Run egg-info command on setup.py. The command generates metadata files.
@@ -182,24 +182,51 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) {
 
 // Get package ID from PKG-INFO file content.
 // If pattern of package name of version not found, return an error.
-func getProjectIdFromFileContent(content []byte) (string, error) {
+func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) {
 	// Create package-name regexp.
-	packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s(\w[\w-.]+)`)
+	packageNameWithPrefixRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp)
 
 	// Find first nameMatch of packageNameRegexp.
-	nameMatch := packageNameRegexp.FindStringSubmatch(string(content))
+	nameMatch := packageNameWithPrefixRegexp.FindStringSubmatch(string(content))
 	if len(nameMatch) < 2 {
-		return "", errors.New("failed extracting package name from content")
+		return "", "", errors.New("failed extracting package name from content")
 	}
 
 	// Create package-version regexp.
-	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s(\w[\w-.]+)`)
+	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + packageNameRegexp)
 
 	// Find first match of packageNameRegexp.
 	versionMatch := packageVersionRegexp.FindStringSubmatch(string(content))
 	if len(versionMatch) < 2 {
-		return "", errors.New("failed extracting package version from content")
+		return "", "", errors.New("failed extracting package version from content")
 	}
 
-	return nameMatch[1] + ":" + versionMatch[1], nil
+	return nameMatch[1], versionMatch[1], nil
+}
+
+// Try getting the name and version from pyproject.toml or from setup.py, if those exist.
+func getPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) {
+	projectName, projectVersion, err = getPipProjectDetailsFromPyProjectToml(srcPath)
+	if err != nil || projectName != "" {
+		return
+	}
+	return getPackageDetailsFromSetuppy(srcPath)
+}
+
+// Returns project ID based on name and version from pyproject.toml or setup.py, if found.
+func getPipProjectId(srcPath string) (string, error) {
+	projectName, projectVersion, err := getPipProjectNameAndVersion(srcPath)
+	if err != nil || projectName == "" {
+		return "", err
+	}
+	return projectName + ":" + projectVersion, nil
+}
+
+// Try getting the name and version from pyproject.toml.
+func getPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) {
+	filePath, err := getPyProjectFilePath(srcPath)
+	if err != nil || filePath == "" {
+		return
+	}
+	return extractPipProjectDetailsFromPyProjectToml(filePath)
 }

utils/pythonutils/piputils_test.go

@@ -9,24 +9,22 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetProjectNameFromFileContent(t *testing.T) {
-	tests := []struct {
-		fileContent         string
-		expectedProjectName string
+func TestGetProjectNameAndVersionFromFileContent(t *testing.T) {
+	testCases := []struct {
+		fileContent            string
+		expectedProjectName    string
+		expectedProjectVersion string
 	}{
-		{"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1:1.0"},
-		{"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]", "jfrog-python-example-2:1.0"},
-		{"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nName: jfrog-python-example-4", "jfrog-python-example-3:1.0"},
+		{"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1", "1.0"},
+		{"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]", "jfrog-python-example-2", "1.0"},
+		{"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nName: jfrog-python-example-4", "jfrog-python-example-3", "1.0"},
 	}
 
-	for _, test := range tests {
-		actualValue, err := getProjectIdFromFileContent([]byte(test.fileContent))
-		if err != nil {
-			t.Error(err)
-		}
-		if actualValue != test.expectedProjectName {
-			t.Errorf("Expected value: %s, got: %s.", test.expectedProjectName, actualValue)
-		}
+	for _, test := range testCases {
+		projectName, projectVersion, err := getProjectNameAndVersionFromFileContent([]byte(test.fileContent))
+		assert.NoError(t, err)
+		assert.Equal(t, test.expectedProjectName, projectName)
+		assert.Equal(t, test.expectedProjectVersion, projectVersion)
 	}
 }
 
@@ -40,16 +38,18 @@ var moduleNameTestProvider = []struct {
 	{"setuppyproject", "overidden-module", "overidden-module", "jfrog-python-example:1.0"},
 	{"requirementsproject", "", "", ""},
 	{"requirementsproject", "overidden-module", "overidden-module", ""},
+	{"pyproject", "", "jfrog-python-example:1.0", "pip-project-with-pyproject:1.2.3"},
+	{"pyproject", "overidden-module", "overidden-module", "pip-project-with-pyproject:1.2.3"},
 }
 
-func TestDetermineModuleName(t *testing.T) {
+func TestGetPipProjectId(t *testing.T) {
 	for _, test := range moduleNameTestProvider {
 		t.Run(strings.Join([]string{test.projectName, test.moduleName}, "/"), func(t *testing.T) {
 			tmpProjectPath, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "pip", test.projectName))
 			defer cleanup()
 
 			// Determine module name
-			packageName, err := getPackageNameFromSetuppy(tmpProjectPath)
+			packageName, err := getPipProjectId(tmpProjectPath)
 			if assert.NoError(t, err) {
 				assert.Equal(t, test.expectedPackageName, packageName)
 			}