From a131da7e40424f6b7dacec756ecc18fd227e3680 Mon Sep 17 00:00:00 2001 From: RobiNino Date: Wed, 18 Sep 2024 11:03:33 +0300 Subject: [PATCH 1/4] Add twine support --- build/build.go | 8 +++ build/golang.go | 6 +-- build/npm.go | 6 +-- build/python.go | 52 +++++++++++------- build/yarn.go | 6 +-- utils/pythonutils/piputils.go | 57 ++++++++++++++------ utils/pythonutils/piputils_test.go | 34 ++++++------ utils/pythonutils/poetryutils.go | 34 ++++-------- utils/pythonutils/poetryutils_test.go | 2 +- utils/pythonutils/pyprojectutils.go | 45 ++++++++++++++++ utils/pythonutils/pyprojectutils_test.go | 35 ++++++++++++ utils/pythonutils/twineutils.go | 59 +++++++++++++++++++++ utils/pythonutils/twineutils_test.go | 45 ++++++++++++++++ utils/pythonutils/utils.go | 18 ++++--- utils/testdata/pip/pyproject/pyproject.toml | 15 ++++++ 15 files changed, 323 insertions(+), 99 deletions(-) create mode 100644 utils/pythonutils/pyprojectutils.go create mode 100644 utils/pythonutils/pyprojectutils_test.go create mode 100644 utils/pythonutils/twineutils.go create mode 100644 utils/pythonutils/twineutils_test.go create mode 100644 utils/testdata/pip/pyproject/pyproject.toml diff --git a/build/build.go b/build/build.go index 10b3aaef..72d872d3 100644 --- a/build/build.go +++ b/build/build.go @@ -373,6 +373,14 @@ func (b *Build) GetBuildTimestamp() time.Time { return b.buildTimestamp } +func (b *Build) AddArtifacts(moduleId string, moduleType entities.ModuleType, artifacts ...entities.Artifact) error { + if !b.buildNameAndNumberProvided() { + return errors.New("a build name must be provided in order to add artifacts") + } + partial := &entities.Partial{ModuleId: moduleId, ModuleType: moduleType, Artifacts: artifacts} + return b.SavePartialBuildInfo(partial) +} + type partialModule struct { moduleType entities.ModuleType artifacts map[string]entities.Artifact diff --git a/build/golang.go b/build/golang.go index 1ccbe207..5dbcf5df 100644 --- a/build/golang.go +++ b/build/golang.go @@ -55,11 +55,7 @@ func (gm *GoModule) SetName(name string) { } func (gm *GoModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !gm.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: gm.name, ModuleType: entities.Go, Artifacts: artifacts} - return gm.containingBuild.SavePartialBuildInfo(partial) + return gm.containingBuild.AddArtifacts(gm.name, entities.Go, artifacts...) } func (gm *GoModule) loadDependencies() ([]entities.Dependency, error) { diff --git a/build/npm.go b/build/npm.go index 6fe77db7..d5b7d0bb 100644 --- a/build/npm.go +++ b/build/npm.go @@ -97,11 +97,7 @@ func (nm *NpmModule) SetCollectBuildInfo(collectBuildInfo bool) { } func (nm *NpmModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !nm.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: nm.name, ModuleType: entities.Npm, Artifacts: artifacts} - return nm.containingBuild.SavePartialBuildInfo(partial) + return nm.containingBuild.AddArtifacts(nm.name, entities.Npm, artifacts...) } // This function discards the npm command in npmArgs and keeps only the command flags. diff --git a/build/python.go b/build/python.go index 60299dc2..df83ae15 100644 --- a/build/python.go +++ b/build/python.go @@ -11,7 +11,7 @@ import ( type PythonModule struct { containingBuild *Build tool pythonutils.PythonTool - name string + id string srcPath string localDependenciesPath string updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error @@ -37,33 +37,40 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e if err != nil { return fmt.Errorf("failed while attempting to get %s dependencies graph: %s", pm.tool, err.Error()) } - // Get package-name. - packageName, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath) - if pkgNameErr != nil { - pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error()) - } - // If module-name was set by the command, don't change it. - if pm.name == "" { - // If the package name is unknown, set the module name to be the build name. - pm.name = packageName - if pm.name == "" { - pm.name = pm.containingBuild.buildName - pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.name)) - } - } + + packageId := pm.SetModuleId() + if pm.updateDepsChecksumInfoFunc != nil { err = pm.updateDepsChecksumInfoFunc(dependenciesMap, pm.srcPath) if err != nil { return err } } - pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageName, pm.name) - buildInfoModule := entities.Module{Id: pm.name, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)} + pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageId, pm.id) + buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)} buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}} return pm.containingBuild.SaveBuildInfo(buildInfo) } +// Sets the module ID and returns the package ID (if found). +func (pm *PythonModule) SetModuleId() (packageId string) { + packageId, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath) + if pkgNameErr != nil { + pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error()) + } + // If module-name was set by the command, don't change it. + if pm.id == "" { + // If the package name is unknown, set the module name to be the build name. + pm.id = packageId + if pm.id == "" { + pm.id = pm.containingBuild.buildName + pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.id)) + } + } + return +} + // Run install command while parsing the logs for downloaded packages. // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...). func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) { @@ -71,7 +78,7 @@ func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string] } func (pm *PythonModule) SetName(name string) { - pm.name = name + pm.id = name } func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) { @@ -81,3 +88,12 @@ func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) { func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error) { pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc } + +func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]string, error) { + pm.SetModuleId() + return pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath) +} + +func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error { + return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...) +} diff --git a/build/yarn.go b/build/yarn.go index fcd436ac..bf3bd1be 100644 --- a/build/yarn.go +++ b/build/yarn.go @@ -149,11 +149,7 @@ func (ym *YarnModule) SetTraverseDependenciesFunc(traverseDependenciesFunc func( } func (ym *YarnModule) AddArtifacts(artifacts ...entities.Artifact) error { - if !ym.containingBuild.buildNameAndNumberProvided() { - return errors.New("a build name must be provided in order to add artifacts") - } - partial := &entities.Partial{ModuleId: ym.name, ModuleType: entities.Npm, Artifacts: artifacts} - return ym.containingBuild.SavePartialBuildInfo(partial) + return ym.containingBuild.AddArtifacts(ym.name, entities.Npm, artifacts...) } func validateYarnVersion(executablePath, srcPath string) error { diff --git a/utils/pythonutils/piputils.go b/utils/pythonutils/piputils.go index b420aa19..4e218617 100644 --- a/utils/pythonutils/piputils.go +++ b/utils/pythonutils/piputils.go @@ -73,20 +73,20 @@ func writeScriptIfNeeded(targetDirPath, scriptName string) error { return nil } -func getPackageNameFromSetuppy(srcPath string) (string, error) { +func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVersion string, err error) { filePath, err := getSetupPyFilePath(srcPath) if err != nil || filePath == "" { // Error was returned or setup.py does not exist in directory. - return "", err + return } // Extract package name from setup.py. - packageName, err := ExtractPackageNameFromSetupPy(filePath) + packageName, packageVersion, err = ExtractPackageNameFromSetupPy(filePath) if err != nil { // If setup.py egg_info command failed we use build name as module name and continue to pip-install execution - return "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error()) + return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error()) } - return packageName, nil + return packageName, packageVersion, nil } // Look for 'setup.py' file in current work dir. @@ -95,16 +95,16 @@ func getSetupPyFilePath(srcPath string) (string, error) { return getFilePath(srcPath, "setup.py") } -// Get the project-name by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file. -func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, error) { +// Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file. +func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) { // Execute egg_info command and return PKG-INFO content. content, err := getEgginfoPkginfoContent(setuppyFilePath) if err != nil { - return "", err + return "", "", err } // Extract project name from file content. - return getProjectIdFromFileContent(content) + return getProjectNameAndVersionFromFileContent(content) } // Run egg-info command on setup.py. The command generates metadata files. @@ -182,24 +182,51 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) { // Get package ID from PKG-INFO file content. // If pattern of package name of version not found, return an error. -func getProjectIdFromFileContent(content []byte) (string, error) { +func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) { // Create package-name regexp. - packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s(\w[\w-.]+)`) + packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s` + _packageNameRegexp) // Find first nameMatch of packageNameRegexp. nameMatch := packageNameRegexp.FindStringSubmatch(string(content)) if len(nameMatch) < 2 { - return "", errors.New("failed extracting package name from content") + return "", "", errors.New("failed extracting package name from content") } // Create package-version regexp. - packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s(\w[\w-.]+)`) + packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + _packageNameRegexp) // Find first match of packageNameRegexp. versionMatch := packageVersionRegexp.FindStringSubmatch(string(content)) if len(versionMatch) < 2 { - return "", errors.New("failed extracting package version from content") + return "", "", errors.New("failed extracting package version from content") } - return nameMatch[1] + ":" + versionMatch[1], nil + return nameMatch[1], versionMatch[1], nil +} + +// Try getting the name and version from pyproject.toml or from setup.py, if those exist. +func GetPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) { + projectName, projectVersion, err = GetPipProjectDetailsFromPyProjectToml(srcPath) + if err != nil || projectName != "" { + return + } + return getPackageDetailsFromSetuppy(srcPath) +} + +// Returns project ID based on name and version from pyproject.toml or setup.py, if found. +func getPipProjectId(srcPath string) (string, error) { + projectName, projectVersion, err := GetPipProjectNameAndVersion(srcPath) + if err != nil || projectName == "" { + return "", err + } + return projectName + ":" + projectVersion, nil +} + +// Try getting the name and version from pyproject.toml. +func GetPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) { + filePath, err := getPyProjectFilePath(srcPath) + if err != nil || filePath == "" { + return + } + return extractPipProjectDetailsFromPyProjectToml(filePath) } diff --git a/utils/pythonutils/piputils_test.go b/utils/pythonutils/piputils_test.go index d2f0064c..93a5e06f 100644 --- a/utils/pythonutils/piputils_test.go +++ b/utils/pythonutils/piputils_test.go @@ -9,24 +9,22 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGetProjectNameFromFileContent(t *testing.T) { - tests := []struct { - fileContent string - expectedProjectName string +func TestGetProjectNameAndVersionFromFileContent(t *testing.T) { + testCases := []struct { + fileContent string + expectedProjectName string + expectedProjectVersion string }{ - {"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1:1.0"}, - {"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com", "jfrog-python-example-2:1.0"}, - {"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com\nName: jfrog-python-example-4", "jfrog-python-example-3:1.0"}, + {"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1", "1.0"}, + {"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com", "jfrog-python-example-2", "1.0"}, + {"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: jfrog@jfrog.com\nName: jfrog-python-example-4", "jfrog-python-example-3", "1.0"}, } - for _, test := range tests { - actualValue, err := getProjectIdFromFileContent([]byte(test.fileContent)) - if err != nil { - t.Error(err) - } - if actualValue != test.expectedProjectName { - t.Errorf("Expected value: %s, got: %s.", test.expectedProjectName, actualValue) - } + for _, test := range testCases { + projectName, projectVersion, err := getProjectNameAndVersionFromFileContent([]byte(test.fileContent)) + assert.NoError(t, err) + assert.Equal(t, test.expectedProjectName, projectName) + assert.Equal(t, test.expectedProjectVersion, projectVersion) } } @@ -40,16 +38,18 @@ var moduleNameTestProvider = []struct { {"setuppyproject", "overidden-module", "overidden-module", "jfrog-python-example:1.0"}, {"requirementsproject", "", "", ""}, {"requirementsproject", "overidden-module", "overidden-module", ""}, + {"pyproject", "", "jfrog-python-example:1.0", "pip-project-with-pyproject:1.2.3"}, + {"pyproject", "overidden-module", "overidden-module", "pip-project-with-pyproject:1.2.3"}, } -func TestDetermineModuleName(t *testing.T) { +func TestGetPipProjectId(t *testing.T) { for _, test := range moduleNameTestProvider { t.Run(strings.Join([]string{test.projectName, test.moduleName}, "/"), func(t *testing.T) { tmpProjectPath, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "pip", test.projectName)) defer cleanup() // Determine module name - packageName, err := getPackageNameFromSetuppy(tmpProjectPath) + packageName, err := getPipProjectId(tmpProjectPath) if assert.NoError(t, err) { assert.Equal(t, test.expectedPackageName, packageName) } diff --git a/utils/pythonutils/poetryutils.go b/utils/pythonutils/poetryutils.go index 03fe56f8..69b7878a 100644 --- a/utils/pythonutils/poetryutils.go +++ b/utils/pythonutils/poetryutils.go @@ -9,9 +9,6 @@ import ( "golang.org/x/exp/maps" ) -type PyprojectToml struct { - Tool map[string]PoetryPackage -} type PoetryPackage struct { Name string Version string @@ -31,7 +28,7 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep // Error was returned or poetry.lock does not exist in directory. return map[string][]string{}, []string{}, err } - projectName, directDependencies, err := getPackageNameFromPyproject(srcPath) + projectName, directDependencies, err := getPoetryPackageFromPyProject(srcPath) if err != nil { return map[string][]string{}, []string{}, err } @@ -56,49 +53,36 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep return graph, graph[projectName], nil } -func getPackageNameFromPyproject(srcPath string) (string, []string, error) { - filePath, err := getPyprojectFilePath(srcPath) +func getPoetryPackageFromPyProject(srcPath string) (string, []string, error) { + filePath, err := getPyProjectFilePath(srcPath) if err != nil || filePath == "" { - // Error was returned or pyproject.toml does not exist in directory. return "", []string{}, err } - // Extract package name from pyproject.toml. - project, err := extractProjectFromPyproject(filePath) + project, err := extractPoetryPackageFromPyProjectToml(filePath) if err != nil { return "", []string{}, err } return project.Name, append(maps.Keys(project.Dependencies), maps.Keys(project.DevDependencies)...), nil } -// Look for 'pyproject.toml' file in current work dir. -// If found, return its absolute path. -func getPyprojectFilePath(srcPath string) (string, error) { - return getFilePath(srcPath, "pyproject.toml") -} - // Look for 'poetry.lock' file in current work dir. // If found, return its absolute path. func getPoetryLockFilePath(srcPath string) (string, error) { return getFilePath(srcPath, "poetry.lock") } -// Get the project-name by parsing the pyproject.toml file. -func extractProjectFromPyproject(pyprojectFilePath string) (project PoetryPackage, err error) { - content, err := os.ReadFile(pyprojectFilePath) - if err != nil { - return - } - var pyprojectFile PyprojectToml - _, err = toml.Decode(string(content), &pyprojectFile) +// Get poetry package by parsing the pyproject.toml file. +func extractPoetryPackageFromPyProjectToml(pyProjectFilePath string) (project PoetryPackage, err error) { + pyProjectFile, err := decodePyProjectToml(pyProjectFilePath) if err != nil { return } - if poetryProject, ok := pyprojectFile.Tool["poetry"]; ok { + if poetryProject, ok := pyProjectFile.Tool["poetry"]; ok { // Extract project name from file content. poetryProject.Name = poetryProject.Name + ":" + poetryProject.Version return poetryProject, nil } - return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyprojectFilePath) + return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyProjectFilePath) } // Get the project-name by parsing the poetry.lock file diff --git a/utils/pythonutils/poetryutils_test.go b/utils/pythonutils/poetryutils_test.go index 7fb44c3f..99607afb 100644 --- a/utils/pythonutils/poetryutils_test.go +++ b/utils/pythonutils/poetryutils_test.go @@ -24,7 +24,7 @@ func TestGetProjectNameFromPyproject(t *testing.T) { tmpProjectPath, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "poetry", testCase.poetryProject)) defer cleanup() - actualValue, err := extractProjectFromPyproject(filepath.Join(tmpProjectPath, "pyproject.toml")) + actualValue, err := extractPoetryPackageFromPyProjectToml(filepath.Join(tmpProjectPath, "pyproject.toml")) assert.NoError(t, err) if actualValue.Name != testCase.expectedProjectName { t.Errorf("Expected value: %s, got: %s.", testCase.expectedProjectName, actualValue) diff --git a/utils/pythonutils/pyprojectutils.go b/utils/pythonutils/pyprojectutils.go new file mode 100644 index 00000000..81c9f67a --- /dev/null +++ b/utils/pythonutils/pyprojectutils.go @@ -0,0 +1,45 @@ +package pythonutils + +import ( + "github.com/BurntSushi/toml" + "os" +) + +type PyProjectToml struct { + // Represents the [tool.poetry] section in pyproject.toml. + Tool map[string]PoetryPackage + // Represents the [project] section in pyproject.toml, for pypi package managers other than poetry. + Project Project +} + +// Pypi project defined for package managers other than poetry (pip, pipenv, etc...) +type Project struct { + Name string + Version string + Description string +} + +// Get project name and version by parsing the pyproject.toml file. +// Try to extract the project name and version from the [project] section, or if requested also try from [tool.poetry] section. +func extractPipProjectDetailsFromPyProjectToml(pyProjectFilePath string) (projectName, projectVersion string, err error) { + pyProjectFile, err := decodePyProjectToml(pyProjectFilePath) + if err != nil { + return + } + return pyProjectFile.Project.Name, pyProjectFile.Project.Version, nil +} + +func decodePyProjectToml(pyProjectFilePath string) (pyProjectFile PyProjectToml, err error) { + content, err := os.ReadFile(pyProjectFilePath) + if err != nil { + return + } + _, err = toml.Decode(string(content), &pyProjectFile) + return +} + +// Look for 'pyproject.toml' file in current work dir. +// If found, return its absolute path. +func getPyProjectFilePath(srcPath string) (string, error) { + return getFilePath(srcPath, "pyproject.toml") +} diff --git a/utils/pythonutils/pyprojectutils_test.go b/utils/pythonutils/pyprojectutils_test.go new file mode 100644 index 00000000..4801cc1c --- /dev/null +++ b/utils/pythonutils/pyprojectutils_test.go @@ -0,0 +1,35 @@ +package pythonutils + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPyProject(t *testing.T) { + testCases := []struct { + testName string + pyProjectFilePath string + expectedName string + expectedVersion string + errExpected bool + }{ + {"successful", filepath.Join("..", "testdata", "pip", "pyproject"), "pip-project-with-pyproject", "1.2.3", false}, + {"does not contain project section", filepath.Join("..", "testdata", "poetry", "project"), "", "", false}, + {"invalid", filepath.Join("..", "testdata", "pip", "setuppyproject", "setup.py"), "", "", true}, + } + + for _, testCase := range testCases { + t.Run(testCase.testName, func(t *testing.T) { + name, version, err := extractPipProjectDetailsFromPyProjectToml(filepath.Join(testCase.pyProjectFilePath, "pyproject.toml")) + if testCase.errExpected { + assert.Error(t, err) + return + } + assert.Equal(t, testCase.expectedName, name) + assert.Equal(t, testCase.expectedVersion, version) + assert.NoError(t, err) + }) + } +} diff --git a/utils/pythonutils/twineutils.go b/utils/pythonutils/twineutils.go new file mode 100644 index 00000000..b10bed31 --- /dev/null +++ b/utils/pythonutils/twineutils.go @@ -0,0 +1,59 @@ +package pythonutils + +import ( + "fmt" + gofrogcmd "github.com/jfrog/gofrog/io" + "github.com/jfrog/gofrog/log" + "regexp" + "slices" + "strings" +) + +const ( + _twineExeName = "twine" + _twineUploadCmdName = "upload" + _verboseFlag = "--verbose" + _disableProgressFlag = "--disable-progress-bar" +) + +// Run a twine upload and parse artifacts paths from logs. +func TwineUploadWithLogParsing(commandArgs []string, srcPath string) (artifactsPaths []string, err error) { + commandArgs = addRequiredFlags(commandArgs) + uploadCmd := gofrogcmd.NewCommand(_twineExeName, _twineUploadCmdName, commandArgs) + uploadCmd.Dir = srcPath + log.Debug("Running twine command: '", _twineExeName, _twineUploadCmdName, strings.Join(commandArgs, " "), "'with build info collection") + _, errorOut, _, err := gofrogcmd.RunCmdWithOutputParser(uploadCmd, true, getArtifactsParser(&artifactsPaths)) + if err != nil { + return nil, fmt.Errorf("failed running '%s %s %s' command with error: '%s - %s'", _twineExeName, _twineUploadCmdName, strings.Join(commandArgs, " "), err.Error(), errorOut) + } + return +} + +// Enabling verbose and disabling progress bar are required for log parsing. +func addRequiredFlags(commandArgs []string) []string { + for _, flag := range []string{_verboseFlag, _disableProgressFlag} { + if !slices.Contains(commandArgs, flag) { + commandArgs = append(commandArgs, flag) + } + } + return commandArgs +} + +func getArtifactsParser(artifactsPaths *[]string) (parser *gofrogcmd.CmdOutputPattern) { + return &gofrogcmd.CmdOutputPattern{ + // Regexp to catch the paths in lines such as "INFO dist/jfrog_python_example-1.0-py3-none-any.whl (1.6 KB)" + // First part ".+\s" is the line prefix. + // Second part "([^ \t]+)" is the artifact path as a group. + // Third part "\s+\([\d.]+\s+[A-Za-z]{2}\)" is the size and unit, surrounded by parentheses. + RegExp: regexp.MustCompile(`^.+\s([^ \t]+)\s+\([\d.]+\s+[A-Za-z]{2}\)`), + ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + // Check for out of bound results. + if len(pattern.MatchedResults)-1 <= 0 { + log.Debug(fmt.Sprintf("Failed extracting artifact name from line: %s", pattern.Line)) + return pattern.Line, nil + } + *artifactsPaths = append(*artifactsPaths, pattern.MatchedResults[1]) + return pattern.Line, nil + }, + } +} diff --git a/utils/pythonutils/twineutils_test.go b/utils/pythonutils/twineutils_test.go new file mode 100644 index 00000000..ed0ec11b --- /dev/null +++ b/utils/pythonutils/twineutils_test.go @@ -0,0 +1,45 @@ +package pythonutils + +import ( + gofrogcmd "github.com/jfrog/gofrog/io" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestTwineUploadCapture(t *testing.T) { + tests := []struct { + name string + text string + expectedCaptures []string + }{ + { + name: "verbose true", + text: ` +Uploading distributions to https://myplatform.jfrog.io/artifactory/api/pypi/twine-local/ +INFO dist/jfrog_python_example-1.0-py3-none-any.whl (1.6 KB) +INFO dist/jfrog_python_example-1.0.tar.gz (2.4 KB) +INFO username set by command options +INFO password set by command options +INFO username: user +INFO password: +Uploading jfrog_python_example-1.0-py3-none-any.whl +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.5/4.5 kB • 00:00 • ? +INFO Response from https://myplatform.jfrog.io/artifactory/api/pypi/twine-local/: + 200 +Uploading jfrog_python_example-1.0.tar.gz +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 5.3/5.3 kB • 00:00 • ? +INFO Response from https://myplatform.jfrog.io/artifactory/api/pypi/twine-local/: + 200`, + expectedCaptures: []string{"dist/jfrog_python_example-1.0-py3-none-any.whl", + "dist/jfrog_python_example-1.0.tar.gz"}, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + var artifacts []string + runDummyTextStream(t, testCase.text, []*gofrogcmd.CmdOutputPattern{getArtifactsParser(&artifacts)}) + assert.ElementsMatch(t, artifacts, testCase.expectedCaptures) + }) + } +} diff --git a/utils/pythonutils/utils.go b/utils/pythonutils/utils.go index 483b5e03..3d22d82d 100644 --- a/utils/pythonutils/utils.go +++ b/utils/pythonutils/utils.go @@ -18,12 +18,14 @@ const ( Pip PythonTool = "pip" Pipenv PythonTool = "pipenv" Poetry PythonTool = "poetry" + Twine PythonTool = "twine" startDownloadingPattern = `^\s*Downloading\s` downloadingCaptureGroup = `[^\s]*` startUsingCachedPattern = `^\s*Using\scached\s` usingCacheCaptureGroup = `[\S]+` endPattern = `\s\(` + _packageNameRegexp = `(\w[\w-.]+)` ) type PythonTool string @@ -99,10 +101,10 @@ func GetPythonDependencies(tool PythonTool, srcPath, localDependenciesPath strin func GetPackageName(tool PythonTool, srcPath string) (packageName string, err error) { switch tool { - case Pip, Pipenv: - return getPackageNameFromSetuppy(srcPath) + case Pip, Pipenv, Twine: + return getPipProjectId(srcPath) case Poetry: - packageName, _, err = getPackageNameFromPyproject(srcPath) + packageName, _, err = getPoetryPackageFromPyProject(srcPath) return default: return "", errors.New(string(tool) + " commands are not supported.") @@ -148,7 +150,7 @@ func updateDepsIdsAndRequestedBy(parentDependency entities.Dependency, dependenc } childDep.Type = fileType } - // Convert Id field from filename to dependency id + // Convert ID field from filename to dependency id childDep.Id = childId // Reassign map entry with new entry copy dependenciesMap[childName] = childDep @@ -213,7 +215,7 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup, } // Mask the pre-known credentials that are provided as command arguments from logs. -// This function creates a log parser for each credentials argument. +// This function creates a log parser for each credentials' argument. func maskPreKnownCredentials(args []string) (parsers []*gofrogcmd.CmdOutputPattern) { for _, arg := range args { // If this argument is a credentials argument, create a log parser that masks it. @@ -268,14 +270,14 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, installCmd.Dir = srcPath dependenciesMap := map[string]entities.Dependency{} - parsers := []*gofrogcmd.CmdOutputPattern{} + var parsers []*gofrogcmd.CmdOutputPattern var packageName string expectingPackageFilePath := false // Extract downloaded package name. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Collecting\s(\w[\w-.]+)`), + RegExp: regexp.MustCompile(`^Collecting\s` + _packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // If this pattern matched a second time before downloaded-file-name was found, prompt a message. if expectingPackageFilePath { @@ -328,7 +330,7 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, // Extract already installed packages names. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`), + RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s` + _packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // Check for out of bound results. if len(pattern.MatchedResults)-1 < 0 { diff --git a/utils/testdata/pip/pyproject/pyproject.toml b/utils/testdata/pip/pyproject/pyproject.toml new file mode 100644 index 00000000..0cdcd34d --- /dev/null +++ b/utils/testdata/pip/pyproject/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pip-project-with-pyproject" +version = "1.2.3" +description = "Project example for building Python project with JFrog products" +authors = [ + { name="JFrog", email="jfrog@jfrog.com" } +] +dependencies = [ + "PyYAML>3.11", + "nltk" +] \ No newline at end of file From 3f25abf57e9d39b596d43e54cf6b343f70b7e619 Mon Sep 17 00:00:00 2001 From: RobiNino Date: Wed, 18 Sep 2024 12:08:44 +0300 Subject: [PATCH 2/4] Support build info generation --- README.md | 6 ++++++ build/python.go | 19 +++++++++++++++++-- cli/cli.go | 30 ++++++++++++++++++++++++++++++ utils/pythonutils/twineutils.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e1b94206..1a480597 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,12 @@ bi pipenv [pipenv command] [command options] Note: checksums calculation is not yet supported for pipenv projects. +#### twine + +```shell +bi twine [twine command] [command options] +``` + #### Dotnet ```shell diff --git a/build/python.go b/build/python.go index df83ae15..ee036c01 100644 --- a/build/python.go +++ b/build/python.go @@ -89,11 +89,26 @@ func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc } -func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]string, error) { +func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]entities.Artifact, error) { pm.SetModuleId() - return pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath) + artifactsPaths, err := pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath) + if err != nil { + return nil, err + } + return pythonutils.CreateArtifactsFromPaths(artifactsPaths) } func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error { return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...) } + +func (pm *PythonModule) TwineUploadAndGenerateBuild(commandArgs []string) error { + artifacts, err := pm.TwineUploadWithLogParsing(commandArgs) + if err != nil { + return err + } + + buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Artifacts: artifacts} + buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}} + return pm.containingBuild.SaveBuildInfo(buildInfo) +} diff --git a/cli/cli.go b/cli/cli.go index fc8628bd..2c541cf4 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -285,6 +285,36 @@ func GetCommands(logger utils.Log) []*clitool.Command { } }, }, + { + Name: "twine", + Usage: "Generate build-info for a twine project", + UsageText: "bi twine", + Flags: flags, + Action: func(context *clitool.Context) (err error) { + service := build.NewBuildInfoService() + service.SetLogger(logger) + bld, err := service.GetOrCreateBuild("twine-build", "1") + if err != nil { + return + } + defer func() { + err = errors.Join(err, bld.Clean()) + }() + pythonModule, err := bld.AddPythonModule("", pythonutils.Twine) + if err != nil { + return + } + filteredArgs := filterCliFlags(context.Args().Slice(), flags) + if filteredArgs[0] == "upload" { + if err := pythonModule.TwineUploadAndGenerateBuild(filteredArgs[1:]); err != nil { + return err + } + return printBuild(bld, context.String(formatFlag)) + } else { + return exec.Command("twine", filteredArgs[1:]...).Run() + } + }, + }, } } diff --git a/utils/pythonutils/twineutils.go b/utils/pythonutils/twineutils.go index b10bed31..6dcf2667 100644 --- a/utils/pythonutils/twineutils.go +++ b/utils/pythonutils/twineutils.go @@ -2,8 +2,12 @@ package pythonutils import ( "fmt" + "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/gofrog/crypto" gofrogcmd "github.com/jfrog/gofrog/io" "github.com/jfrog/gofrog/log" + "path" + "path/filepath" "regexp" "slices" "strings" @@ -57,3 +61,29 @@ func getArtifactsParser(artifactsPaths *[]string) (parser *gofrogcmd.CmdOutputPa }, } } + +// Create artifacts entities from the artifacts paths that were found during the upload. +func CreateArtifactsFromPaths(artifactsPaths []string) (artifacts []entities.Artifact, err error) { + projectName, projectVersion, err := GetPipProjectNameAndVersion("") + if err != nil { + return + } + var absPath string + var fileDetails *crypto.FileDetails + for _, artifactPath := range artifactsPaths { + absPath, err = filepath.Abs(artifactPath) + if err != nil { + return nil, err + } + fileDetails, err = crypto.GetFileDetails(absPath, true) + if err != nil { + return nil, err + } + + artifact := entities.Artifact{Name: filepath.Base(absPath), Path: path.Join(projectName, projectVersion, filepath.Base(absPath)), + Type: strings.TrimPrefix(filepath.Ext(absPath), ".")} + artifact.Checksum = entities.Checksum{Sha1: fileDetails.Checksum.Sha1, Md5: fileDetails.Checksum.Md5} + artifacts = append(artifacts, artifact) + } + return +} From 8106d1f96be46f24ec25cbb016c8196783d237e2 Mon Sep 17 00:00:00 2001 From: RobiNino Date: Wed, 18 Sep 2024 16:14:48 +0300 Subject: [PATCH 3/4] CR comments --- utils/pythonutils/piputils.go | 16 ++++++++-------- utils/pythonutils/twineutils.go | 2 +- utils/pythonutils/utils.go | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/utils/pythonutils/piputils.go b/utils/pythonutils/piputils.go index 4e218617..4e6593cf 100644 --- a/utils/pythonutils/piputils.go +++ b/utils/pythonutils/piputils.go @@ -81,7 +81,7 @@ func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVe } // Extract package name from setup.py. - packageName, packageVersion, err = ExtractPackageNameFromSetupPy(filePath) + packageName, packageVersion, err = extractPackageNameFromSetupPy(filePath) if err != nil { // If setup.py egg_info command failed we use build name as module name and continue to pip-install execution return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error()) @@ -96,7 +96,7 @@ func getSetupPyFilePath(srcPath string) (string, error) { } // Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file. -func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) { +func extractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) { // Execute egg_info command and return PKG-INFO content. content, err := getEgginfoPkginfoContent(setuppyFilePath) if err != nil { @@ -184,7 +184,7 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) { // If pattern of package name of version not found, return an error. func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) { // Create package-name regexp. - packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s` + _packageNameRegexp) + packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp) // Find first nameMatch of packageNameRegexp. nameMatch := packageNameRegexp.FindStringSubmatch(string(content)) @@ -193,7 +193,7 @@ func getProjectNameAndVersionFromFileContent(content []byte) (string, string, er } // Create package-version regexp. - packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + _packageNameRegexp) + packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + packageNameRegexp) // Find first match of packageNameRegexp. versionMatch := packageVersionRegexp.FindStringSubmatch(string(content)) @@ -205,8 +205,8 @@ func getProjectNameAndVersionFromFileContent(content []byte) (string, string, er } // Try getting the name and version from pyproject.toml or from setup.py, if those exist. -func GetPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) { - projectName, projectVersion, err = GetPipProjectDetailsFromPyProjectToml(srcPath) +func getPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) { + projectName, projectVersion, err = getPipProjectDetailsFromPyProjectToml(srcPath) if err != nil || projectName != "" { return } @@ -215,7 +215,7 @@ func GetPipProjectNameAndVersion(srcPath string) (projectName string, projectVer // Returns project ID based on name and version from pyproject.toml or setup.py, if found. func getPipProjectId(srcPath string) (string, error) { - projectName, projectVersion, err := GetPipProjectNameAndVersion(srcPath) + projectName, projectVersion, err := getPipProjectNameAndVersion(srcPath) if err != nil || projectName == "" { return "", err } @@ -223,7 +223,7 @@ func getPipProjectId(srcPath string) (string, error) { } // Try getting the name and version from pyproject.toml. -func GetPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) { +func getPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) { filePath, err := getPyProjectFilePath(srcPath) if err != nil || filePath == "" { return diff --git a/utils/pythonutils/twineutils.go b/utils/pythonutils/twineutils.go index 6dcf2667..4148d4b9 100644 --- a/utils/pythonutils/twineutils.go +++ b/utils/pythonutils/twineutils.go @@ -64,7 +64,7 @@ func getArtifactsParser(artifactsPaths *[]string) (parser *gofrogcmd.CmdOutputPa // Create artifacts entities from the artifacts paths that were found during the upload. func CreateArtifactsFromPaths(artifactsPaths []string) (artifacts []entities.Artifact, err error) { - projectName, projectVersion, err := GetPipProjectNameAndVersion("") + projectName, projectVersion, err := getPipProjectNameAndVersion("") if err != nil { return } diff --git a/utils/pythonutils/utils.go b/utils/pythonutils/utils.go index 3d22d82d..9db3657b 100644 --- a/utils/pythonutils/utils.go +++ b/utils/pythonutils/utils.go @@ -25,7 +25,7 @@ const ( startUsingCachedPattern = `^\s*Using\scached\s` usingCacheCaptureGroup = `[\S]+` endPattern = `\s\(` - _packageNameRegexp = `(\w[\w-.]+)` + packageNameRegexp = `(\w[\w-.]+)` ) type PythonTool string @@ -277,7 +277,7 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, // Extract downloaded package name. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Collecting\s` + _packageNameRegexp), + RegExp: regexp.MustCompile(`^Collecting\s` + packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // If this pattern matched a second time before downloaded-file-name was found, prompt a message. if expectingPackageFilePath { @@ -330,7 +330,7 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, // Extract already installed packages names. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ - RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s` + _packageNameRegexp), + RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s` + packageNameRegexp), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // Check for out of bound results. if len(pattern.MatchedResults)-1 < 0 { From 0d7d41ddd0e1ae649aaa420a8f1267294e9eeaf0 Mon Sep 17 00:00:00 2001 From: RobiNino Date: Wed, 18 Sep 2024 16:29:25 +0300 Subject: [PATCH 4/4] CR comments --- utils/pythonutils/piputils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/pythonutils/piputils.go b/utils/pythonutils/piputils.go index 4e6593cf..2df46593 100644 --- a/utils/pythonutils/piputils.go +++ b/utils/pythonutils/piputils.go @@ -184,10 +184,10 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) { // If pattern of package name of version not found, return an error. func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) { // Create package-name regexp. - packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp) + packageNameWithPrefixRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp) // Find first nameMatch of packageNameRegexp. - nameMatch := packageNameRegexp.FindStringSubmatch(string(content)) + nameMatch := packageNameWithPrefixRegexp.FindStringSubmatch(string(content)) if len(nameMatch) < 2 { return "", "", errors.New("failed extracting package name from content") }