diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 389eb034..aea43fd9 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -12,32 +12,24 @@ jobs: steps: - name: Checkout Source uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x - cache: false - - name: Static Code Analysis - uses: golangci/golangci-lint-action@v3 - with: - args: | - --timeout 5m --out-${NO_FUTURE}format colored-line-number --enable errcheck,gosimple,govet,ineffassign,staticcheck,typecheck,unused,gocritic,asasalint,asciicheck,errchkjson,exportloopref,forcetypeassert,makezero,nilerr,unparam,unconvert,wastedassign,usestdlibvars - + + - name: Setup Go with cache + uses: jfrog/.github/actions/install-go-with-cache@main + + - name: Run Go-Sec scanner + uses: jfrog/.github/actions/gosec-scanner@main Go-Sec: runs-on: ubuntu-latest steps: - name: Checkout Source uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x - cache: false - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: -exclude G107,G204,G301,G302,G304,G306 -tests -exclude-dir \.*test\.* ./... + + - name: Setup Go with cache + uses: jfrog/.github/actions/install-go-with-cache@main + + - name: Run golangci lint + uses: jfrog/.github/actions/golangci-lint@main ShellCheck: runs-on: ubuntu-latest diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 38183d2d..65f85b19 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -10,26 +10,10 @@ jobs: CLAssistant: runs-on: ubuntu-latest steps: - - uses: actions-ecosystem/action-regex-match@v2 - id: sign-or-recheck + - name: Run CLA Check + uses: jfrog/.github/actions/cla@main with: - text: ${{ github.event.comment.body }} - regex: '\s*(I have read the CLA Document and I hereby sign the CLA)|(recheck)\s*' - - - name: "CLA Assistant" - if: ${{ steps.sign-or-recheck.outputs.match != '' || github.event_name == 'pull_request_target' }} - # Alpha Release - uses: cla-assistant/github-action@v2.3.0 - env: - # Generated and maintained by GitHub + event_comment_body: ${{ github.event.comment.body }} + event_name: ${{ github.event_name }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # JFrog organization secret - PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_SIGN_TOKEN }} - with: - path-to-signatures: 'signed_clas.json' - path-to-document: 'https://jfrog.com/cla/' - remote-organization-name: 'jfrog' - remote-repository-name: 'jfrog-signed-clas' - # branch should not be protected - branch: 'master' - allowlist: bot* + CLA_SIGN_TOKEN: ${{ secrets.CLA_SIGN_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/frogbot-scan-and-fix.yml b/.github/workflows/frogbot-scan-and-fix.yml index 5a14ec18..bc1ede2e 100644 --- a/.github/workflows/frogbot-scan-and-fix.yml +++ b/.github/workflows/frogbot-scan-and-fix.yml @@ -19,11 +19,8 @@ jobs: with: ref: ${{ matrix.branch }} - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x + - name: Setup Go with cache + uses: jfrog/.github/actions/install-go-with-cache@main - uses: jfrog/frogbot@v2 env: diff --git a/.github/workflows/frogbot-scan-pr.yml b/.github/workflows/frogbot-scan-pr.yml index 55b1fea2..fa46baff 100644 --- a/.github/workflows/frogbot-scan-pr.yml +++ b/.github/workflows/frogbot-scan-pr.yml @@ -12,11 +12,8 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x + - name: Setup Go with cache + uses: jfrog/.github/actions/install-go-with-cache@main - uses: jfrog/frogbot@v2 env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c41efeb9..53f1d56a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,11 +36,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x - - name: Setup Python3 uses: actions/setup-python@v5 with: @@ -50,17 +45,13 @@ jobs: run: pip3 install pipenv - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} check-latest: true - - name: Go Cache - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-go- + - name: Setup Go with cache + uses: jfrog/.github/actions/install-go-with-cache@main - name: Lint run: go vet ./... diff --git a/build/golang.go b/build/golang.go index 548bf5e9..1ccbe207 100644 --- a/build/golang.go +++ b/build/golang.go @@ -3,12 +3,12 @@ package build import ( "errors" "fmt" + "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/crypto" "path/filepath" "strings" "unicode" - - "github.com/jfrog/build-info-go/entities" - "github.com/jfrog/build-info-go/utils" ) type GoModule struct { @@ -164,12 +164,12 @@ func (gm *GoModule) getPackagePathIfExists(cachePath, encodedDependencyId string func populateZip(packageId, zipPath string) (zipDependency entities.Dependency, err error) { // Zip file dependency for the build-info zipDependency = entities.Dependency{Id: packageId} - checksums, err := utils.GetFileChecksums(zipPath) + checksums, err := crypto.GetFileChecksums(zipPath) if err != nil { return } zipDependency.Type = "zip" - zipDependency.Checksum = entities.Checksum{Sha1: checksums[utils.SHA1], Md5: checksums[utils.MD5], Sha256: checksums[utils.SHA256]} + zipDependency.Checksum = entities.Checksum{Sha1: checksums[crypto.SHA1], Md5: checksums[crypto.MD5], Sha256: checksums[crypto.SHA256]} return } diff --git a/build/gradle.go b/build/gradle.go index 2e8f917d..ecc5d8dc 100644 --- a/build/gradle.go +++ b/build/gradle.go @@ -21,8 +21,8 @@ const ( gradleExtractorFileName = "build-info-extractor-gradle-%s-uber.jar" gradleInitScriptTemplate = "gradle.init" gradleExtractorRemotePath = "org/jfrog/buildinfo/build-info-extractor-gradle/%s" - gradleExtractor4DependencyVersion = "4.33.13" - gradleExtractor5DependencyVersion = "5.2.0" + gradleExtractor4DependencyVersion = "4.33.22" + gradleExtractor5DependencyVersion = "5.2.5" projectPropertiesFlag = "-P" systemPropertiesFlag = "-D" ) diff --git a/build/maven.go b/build/maven.go index 65c8bba2..970c1394 100644 --- a/build/maven.go +++ b/build/maven.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/jfrog/build-info-go/utils" + "golang.org/x/term" ) const ( @@ -20,7 +21,7 @@ const ( classworldsConfFileName = "classworlds.conf" PropertiesTempFolderName = "properties" MavenExtractorRemotePath = "org/jfrog/buildinfo/build-info-extractor-maven3/%s" - MavenExtractorDependencyVersion = "2.41.16" + MavenExtractorDependencyVersion = "2.41.23" ClassworldsConf = `main is org.apache.maven.cli.MavenCli from plexus.core @@ -43,6 +44,8 @@ type MavenModule struct { extractorDetails *extractorDetails // A pipe to write the maven extractor output to. outputWriter io.Writer + // Path to the build info temp file that will be generated by the maven extractor. + buildInfoPath string } // Maven extractor is the engine for calculating the project dependencies. @@ -111,6 +114,12 @@ func (mm *MavenModule) SetMavenOpts(mavenOpts ...string) { mm.extractorDetails.mavenOpts = mavenOpts } +// Returns the path to the build info generated by the maven extractor. +// This file is a tempfile that can be consumed to generated a build info object. +func (mm *MavenModule) GetGeneratedBuildInfoPath() string { + return mm.buildInfoPath +} + func (mm *MavenModule) createMvnRunConfig() (*mvnRunConfig, error) { var javaExecPath string mavenHome, err := mm.loadMavenHome() @@ -133,11 +142,11 @@ func (mm *MavenModule) createMvnRunConfig() (*mvnRunConfig, error) { if len(plexusClassworlds) != 1 { return nil, errors.New("couldn't find plexus-classworlds-x.x.x.jar in Maven installation path, please check M2_HOME environment variable") } - buildInfoPath, err := createEmptyBuildInfoFile(mm.containingBuild) + mm.buildInfoPath, err = createEmptyBuildInfoFile(mm.containingBuild) if err != nil { return nil, err } - extractorProps, err := utils.CreateExtractorPropsFile(mm.extractorDetails.propsDir, buildInfoPath, mm.containingBuild.buildName, mm.containingBuild.buildNumber, mm.containingBuild.buildTimestamp, mm.containingBuild.projectKey, mm.extractorDetails.props) + extractorProps, err := utils.CreateExtractorPropsFile(mm.extractorDetails.propsDir, mm.buildInfoPath, mm.containingBuild.buildName, mm.containingBuild.buildNumber, mm.containingBuild.buildTimestamp, mm.containingBuild.projectKey, mm.extractorDetails.props) if err != nil { return nil, err } @@ -332,15 +341,41 @@ func (config *mvnRunConfig) SetOutputWriter(outputWriter io.Writer) *mvnRunConfi return config } -func (config *mvnRunConfig) runCmd() error { +func (config *mvnRunConfig) runCmd() (err error) { command := config.GetCmd() - command.Stderr = os.Stderr + errBuffer := bytes.NewBuffer([]byte{}) + multiWriter := io.MultiWriter(os.Stderr, errBuffer) + command.Stderr = multiWriter if config.outputWriter == nil { command.Stdout = os.Stderr } else { command.Stdout = config.outputWriter } command.Dir = config.workspace + addColorToCmdOutput(command) config.logger.Info("Running mvn command:", strings.Join(command.Args, " ")) - return command.Run() + + err = command.Run() + if err != nil { + if utils.IsForbiddenOutput(utils.Maven, errBuffer.String()) { + err = errors.Join(utils.NewForbiddenError(), err) + } + } + return +} + +// To always have color in Maven's output, add "-Dstyle.color=always" to the command line arguments +func addColorToCmdOutput(command *exec.Cmd) { + if term.IsTerminal(int(os.Stderr.Fd())) { + shouldAddColor := true + for _, arg := range command.Args { + if strings.Contains(arg, "-Dstyle.color") { + shouldAddColor = false + break + } + } + if shouldAddColor { + command.Args = append(command.Args, "-Dstyle.color=always") + } + } } diff --git a/build/maven_test.go b/build/maven_test.go index 75d514f2..a6cb8f60 100644 --- a/build/maven_test.go +++ b/build/maven_test.go @@ -8,7 +8,9 @@ import ( "github.com/jfrog/build-info-go/tests" "github.com/jfrog/build-info-go/utils" "os" + "os/exec" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -127,3 +129,54 @@ func TestGetExecutableName(t *testing.T) { assert.Equal(t, result, mvnHome) } } + +func TestAddColorToCmdOutput(t *testing.T) { + testCases := []struct { + name string + initialArgs []string + expectedResult string + colorArgExist bool + }{ + { + name: "Not a terminal, shouldn't add color", + initialArgs: []string{"mvn"}, + colorArgExist: false, + }, + { + name: "Terminal supports color and existing color argument", + initialArgs: []string{"mvn", "-Dstyle.color=always"}, + expectedResult: "Dstyle.color=always", + colorArgExist: true, + }, + { + name: "Terminal supports color and existing color argument", + initialArgs: []string{"mvn", "-Dstyle.color=never"}, + expectedResult: "Dstyle.color=never", + colorArgExist: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Mock terminal support + + // Create a mock exec.Cmd object + cmd := exec.Command(tc.initialArgs[0], tc.initialArgs[1:]...) + + // Call the function to test + addColorToCmdOutput(cmd) + + // Check if the argument was added + containsColorArg := false + for _, arg := range cmd.Args { + if strings.Contains(arg, "Dstyle.color") { + if strings.Contains(arg, tc.expectedResult) { + containsColorArg = true + break + } + } + } + assert.Equal(t, tc.colorArgExist, containsColorArg) + }) + } +} diff --git a/build/utils/dotnet/dependencies/assetsjson.go b/build/utils/dotnet/dependencies/assetsjson.go index b0708e63..53b85f63 100644 --- a/build/utils/dotnet/dependencies/assetsjson.go +++ b/build/utils/dotnet/dependencies/assetsjson.go @@ -6,6 +6,7 @@ import ( "fmt" buildinfo "github.com/jfrog/build-info-go/entities" "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/crypto" "os" "path/filepath" "strings" @@ -111,7 +112,7 @@ func (assets *assets) getAllDependencies(log utils.Log) (map[string]*buildinfo.D } return nil, errors.New("The file " + nupkgFilePath + " doesn't exist in the NuGet cache directory.") } - fileDetails, err := utils.GetFileDetails(nupkgFilePath, true) + fileDetails, err := crypto.GetFileDetails(nupkgFilePath, true) if err != nil { return nil, err } diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index 094e3062..9445877e 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -4,14 +4,17 @@ import ( "encoding/binary" "encoding/xml" "fmt" - "github.com/jfrog/build-info-go/build/utils/dotnet" - buildinfo "github.com/jfrog/build-info-go/entities" - "github.com/jfrog/build-info-go/utils" - gofrogcmd "github.com/jfrog/gofrog/io" "os" "path/filepath" "strings" "unicode/utf16" + + "github.com/jfrog/gofrog/crypto" + gofrogcmd "github.com/jfrog/gofrog/io" + + "github.com/jfrog/build-info-go/build/utils/dotnet" + buildinfo "github.com/jfrog/build-info-go/entities" + "github.com/jfrog/build-info-go/utils" ) const ( @@ -78,7 +81,7 @@ func (extractor *packagesExtractor) extract(packagesConfig *packagesConfig, glob } if pack == nil { // If it doesn't exist lets build the array of alternative versions. - alternativeVersions := createAlternativeVersionForms(nuget.Version) + alternativeVersions := CreateAlternativeVersionForms(nuget.Version) // Now let's do a loop to run over the alternative possibilities for i := 0; i < len(alternativeVersions); i++ { nPackage.version = alternativeVersions[i] @@ -109,7 +112,7 @@ func (extractor *packagesExtractor) extract(packagesConfig *packagesConfig, glob // "1.22.33" --> []string{"1.22.33.0"} // "1.22.33.44" --> []string{} // "1.0.2" --> []string{"1.0.2.0"} -func createAlternativeVersionForms(originalVersion string) []string { +func CreateAlternativeVersionForms(originalVersion string) []string { versionSlice := strings.Split(originalVersion, ".") versionSliceSize := len(versionSlice) for i := 4; i > versionSliceSize; i-- { @@ -210,7 +213,7 @@ func createNugetPackage(packagesPath string, nuget xmlPackage, nPackage *nugetPa return nil, nil } - fileDetails, err := utils.GetFileDetails(nupkgPath, true) + fileDetails, err := crypto.GetFileDetails(nupkgPath, true) if err != nil { return nil, err } diff --git a/build/utils/dotnet/dependencies/packagesconfig_test.go b/build/utils/dotnet/dependencies/packagesconfig_test.go index 5b849972..9a656890 100644 --- a/build/utils/dotnet/dependencies/packagesconfig_test.go +++ b/build/utils/dotnet/dependencies/packagesconfig_test.go @@ -61,7 +61,7 @@ func TestAlternativeVersionsForms(t *testing.T) { } for _, test := range tests { t.Run(test.version, func(t *testing.T) { - actual := createAlternativeVersionForms(test.version) + actual := CreateAlternativeVersionForms(test.version) sort.Strings(actual) sort.Strings(test.expected) if len(actual) != len(test.expected) { diff --git a/build/utils/npm.go b/build/utils/npm.go index 6de62542..d697c4e3 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/jfrog/gofrog/crypto" "os" "os/exec" "path/filepath" @@ -381,11 +382,11 @@ func calculateChecksum(cacache *cacache, name, version, integrity string) (md5 s if err != nil { return } - checksums, err := utils.GetFileChecksums(path) + checksums, err := crypto.GetFileChecksums(path) if err != nil { return } - return checksums[utils.MD5], checksums[utils.SHA1], checksums[utils.SHA256], err + return checksums[crypto.MD5], checksums[crypto.SHA1], checksums[crypto.SHA256], err } // Merge two scopes and remove duplicates. diff --git a/build/utils/npmcacache.go b/build/utils/npmcacache.go index 14452052..c264c691 100644 --- a/build/utils/npmcacache.go +++ b/build/utils/npmcacache.go @@ -5,10 +5,10 @@ import ( "encoding/hex" "encoding/json" "errors" + "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/crypto" "path/filepath" "strings" - - "github.com/jfrog/build-info-go/utils" ) // npm stores cache data in an opaque directory within the configured cache, named _cacache. @@ -105,11 +105,11 @@ func (c *cacache) GetInfo(id string) (*cacacheInfo, error) { // "pacote:tarball:ansi-regex@5.0.0" -> // ~/.my-cache/index-v5/4e/22/eb8971d3255ba68fad66a1be245aaf480c23e8b02cf0dae7022549aece7c func (c *cacache) getIndexByKey(key string) (string, error) { - hashMap, err := utils.CalcChecksums(strings.NewReader(key), utils.SHA256) + hashMap, err := crypto.CalcChecksums(strings.NewReader(key), crypto.SHA256) if err != nil { return "", err } - hashedKey := hashMap[utils.SHA256] + hashedKey := hashMap[crypto.SHA256] return c.getIndexByHash(hashedKey) } diff --git a/entities/buildinfo.go b/entities/buildinfo.go index 6f0144bb..f330ac54 100644 --- a/entities/buildinfo.go +++ b/entities/buildinfo.go @@ -355,6 +355,9 @@ type Module struct { Artifacts []Artifact `json:"artifacts,omitempty"` ExcludedArtifacts []Artifact `json:"excludedArtifacts,omitempty"` Dependencies []Dependency `json:"dependencies,omitempty"` + // Parent is used to store the parent module id, for multi-module projects. + // This field is not recognized by Artifactory, and is used for internal purposes only. + Parent string `json:"parent,omitempty"` // Used in aggregated builds - this field stores the checksums of the referenced build-info JSON. Checksum } @@ -420,6 +423,10 @@ type Artifact struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Path string `json:"path,omitempty"` + // The target repository to which the artifact was deployed to. + // Named 'original' because the repository might change throughout the lifecycle of the build. + // This field is not recognized by Artifactory, and is used for internal purposes only. + OriginalDeploymentRepo string `json:"originalDeploymentRepo,omitempty"` Checksum } diff --git a/go.mod b/go.mod index cd3e901b..66ed1f3e 100644 --- a/go.mod +++ b/go.mod @@ -3,27 +3,32 @@ module github.com/jfrog/build-info-go go 1.22 require ( - github.com/BurntSushi/toml v1.3.2 - github.com/CycloneDX/cyclonedx-go v0.8.0 + github.com/BurntSushi/toml v1.4.0 + github.com/CycloneDX/cyclonedx-go v0.9.0 github.com/buger/jsonparser v1.1.1 - github.com/jfrog/gofrog v1.7.2 - github.com/minio/sha256-simd v1.0.1 + github.com/jfrog/gofrog v1.7.6 github.com/stretchr/testify v1.9.0 - github.com/urfave/cli/v2 v2.27.1 + github.com/urfave/cli/v2 v2.27.4 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e + golang.org/x/term v0.24.0 ) require ( - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gookit/color v1.5.4 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect + github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.7.6-0.20240909061051-2d36ae4bd05a diff --git a/go.sum b/go.sum index 61571a4d..c8207e43 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,20 @@ -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= -github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/CycloneDX/cyclonedx-go v0.9.0 h1:inaif7qD8bivyxp7XLgxUYtOXWtDez7+j72qKTMQTb8= +github.com/CycloneDX/cyclonedx-go v0.9.0/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/jfrog/gofrog v1.7.2 h1:VkAaA/9tmbw27IqgUOmaZWnO6ATUqL3vRzDnsROKATw= -github.com/jfrog/gofrog v1.7.2/go.mod h1:WJFk88SR9Sr9mKl1bQBig7DmSdXiBGKV3WhL9O6jL9w= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s= +github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -27,23 +29,27 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= -github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= -github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8= +github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/utils/checksum.go b/utils/checksum.go deleted file mode 100644 index b748fa54..00000000 --- a/utils/checksum.go +++ /dev/null @@ -1,108 +0,0 @@ -package utils - -import ( - "bufio" - //#nosec G501 -- md5 is supported by Artifactory. - "crypto/md5" - //#nosec G505 -- sha1 is supported by Artifactory. - "crypto/sha1" - "fmt" - ioutils "github.com/jfrog/gofrog/io" - "github.com/minio/sha256-simd" - "hash" - "io" - "os" -) - -type Algorithm int - -const ( - MD5 Algorithm = iota - SHA1 - SHA256 -) - -var algorithmFunc = map[Algorithm]func() hash.Hash{ - // Go native crypto algorithms: - MD5: md5.New, - SHA1: sha1.New, - // sha256-simd algorithm: - SHA256: sha256.New, -} - -func GetFileChecksums(filePath string, checksumType ...Algorithm) (checksums map[Algorithm]string, err error) { - file, err := os.Open(filePath) - if err != nil { - return - } - defer ioutils.Close(file, &err) - return CalcChecksums(file, checksumType...) -} - -// CalcChecksums calculates all hashes at once using AsyncMultiWriter. The file is therefore read only once. -func CalcChecksums(reader io.Reader, checksumType ...Algorithm) (map[Algorithm]string, error) { - hashes, err := calcChecksums(reader, checksumType...) - if err != nil { - return nil, err - } - results := sumResults(hashes) - return results, nil -} - -// CalcChecksumsBytes calculates hashes like `CalcChecksums`, returns result as bytes -func CalcChecksumsBytes(reader io.Reader, checksumType ...Algorithm) (map[Algorithm][]byte, error) { - hashes, err := calcChecksums(reader, checksumType...) - if err != nil { - return nil, err - } - results := sumResultsBytes(hashes) - return results, nil -} - -func calcChecksums(reader io.Reader, checksumType ...Algorithm) (map[Algorithm]hash.Hash, error) { - hashes := getChecksumByAlgorithm(checksumType...) - var multiWriter io.Writer - pageSize := os.Getpagesize() - sizedReader := bufio.NewReaderSize(reader, pageSize) - var hashWriter []io.Writer - for _, v := range hashes { - hashWriter = append(hashWriter, v) - } - multiWriter = AsyncMultiWriter(hashWriter...) - _, err := io.Copy(multiWriter, sizedReader) - if err != nil { - return nil, err - } - return hashes, nil -} - -func sumResults(hashes map[Algorithm]hash.Hash) map[Algorithm]string { - results := map[Algorithm]string{} - for k, v := range hashes { - results[k] = fmt.Sprintf("%x", v.Sum(nil)) - } - return results -} - -func sumResultsBytes(hashes map[Algorithm]hash.Hash) map[Algorithm][]byte { - results := map[Algorithm][]byte{} - for k, v := range hashes { - results[k] = v.Sum(nil) - } - return results -} - -func getChecksumByAlgorithm(checksumType ...Algorithm) map[Algorithm]hash.Hash { - hashes := map[Algorithm]hash.Hash{} - if len(checksumType) == 0 { - for k, v := range algorithmFunc { - hashes[k] = v() - } - return hashes - } - - for _, v := range checksumType { - hashes[v] = algorithmFunc[v]() - } - return hashes -} diff --git a/utils/checksum_test.go b/utils/checksum_test.go deleted file mode 100644 index 6c9c6c52..00000000 --- a/utils/checksum_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package utils - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - fileContent = "Why did the robot bring a ladder to the bar? It heard the drinks were on the house." - expectedMd5 = "70bd6370a86813f2504020281e4a2e2e" - expectedSha1 = "8c3578ac814c9f02803001a5d3e5d78a7fd0f9cc" - expectedSha256 = "093d901b28a59f7d95921f3f4fb97a03fe7a1cf8670507ffb1d6f9a01b3e890a" -) - -func TestGetFileChecksums(t *testing.T) { - // Create a temporary file - tempFile, err := os.CreateTemp("", "TestGetFileChecksums") - assert.NoError(t, err) - defer func() { - assert.NoError(t, tempFile.Close()) - assert.NoError(t, os.Remove(tempFile.Name())) - }() - - // Write something to the file - _, err = tempFile.Write([]byte(fileContent)) - assert.NoError(t, err) - - // Calculate only sha1 and match - checksums, err := GetFileChecksums(tempFile.Name(), SHA1) - assert.NoError(t, err) - assert.Len(t, checksums, 1) - assert.Equal(t, expectedSha1, checksums[SHA1]) - - // Calculate md5, sha1 and sha256 checksums and match - checksums, err = GetFileChecksums(tempFile.Name()) - assert.NoError(t, err) - assert.Equal(t, expectedMd5, checksums[MD5]) - assert.Equal(t, expectedSha1, checksums[SHA1]) - assert.Equal(t, expectedSha256, checksums[SHA256]) -} diff --git a/utils/error.go b/utils/error.go new file mode 100644 index 00000000..8d14cbd1 --- /dev/null +++ b/utils/error.go @@ -0,0 +1,48 @@ +package utils + +import ( + "strings" +) + +type PackageManager string + +const ( + Npm PackageManager = "npm" + Maven PackageManager = "maven" + Pip PackageManager = "pip" + Go PackageManager = "go" +) + +// ForbiddenError represents a 403 Forbidden error. +type ForbiddenError struct { + Message string +} + +// Error implements the error interface for ForbiddenError. +func (e *ForbiddenError) Error() string { + return "403 Forbidden" +} + +// NewForbiddenError creates a new ForbiddenError with the given message. +func NewForbiddenError() *ForbiddenError { + return &ForbiddenError{} +} + +// IsForbiddenOutput checks whether the provided output includes a 403 Forbidden. The various package managers have their own forbidden output formats. +func IsForbiddenOutput(tech PackageManager, cmdOutput string) bool { + switch tech { + case "npm": + return strings.Contains(strings.ToLower(cmdOutput), "403 forbidden") + case "maven": + return strings.Contains(cmdOutput, "status code: 403") || + strings.Contains(strings.ToLower(cmdOutput), "403 forbidden") || + // In some cases mvn returns 500 status code even though it got 403 from artifactory. + strings.Contains(cmdOutput, "status code: 500") + case "pip": + return strings.Contains(strings.ToLower(cmdOutput), "http error 403") + case "go": + return strings.Contains(strings.ToLower(cmdOutput), "403 forbidden") || + strings.Contains(strings.ToLower(cmdOutput), " 403") + } + return false +} diff --git a/utils/fileutils.go b/utils/fileutils.go index 61d766ac..4509c7dd 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - ioutils "github.com/jfrog/gofrog/io" "golang.org/x/exp/slices" "io" "net/http" @@ -224,8 +223,14 @@ func ListFilesByFilterFunc(path string, filterFunc func(filePath string) (bool, } func DownloadFile(downloadTo string, fromUrl string) (err error) { - // Get the data - resp, err := http.Get(fromUrl) + req, err := http.NewRequest(http.MethodGet, fromUrl, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } if err != nil { return } @@ -564,42 +569,3 @@ type FileDetails struct { Checksum entities.Checksum Size int64 } - -func GetFileDetails(filePath string, includeChecksums bool) (details *FileDetails, err error) { - details = new(FileDetails) - if includeChecksums { - details.Checksum, err = calcChecksumDetails(filePath) - if err != nil { - return - } - } else { - details.Checksum = entities.Checksum{} - } - - file, err := os.Open(filePath) - defer ioutils.Close(file, &err) - if err != nil { - return - } - fileInfo, err := file.Stat() - if err != nil { - return - } - details.Size = fileInfo.Size() - return -} - -func calcChecksumDetails(filePath string) (checksum entities.Checksum, err error) { - file, err := os.Open(filePath) - if err != nil { - return - } - defer ioutils.Close(file, &err) - - checksums, err := CalcChecksums(file) - if err != nil { - return entities.Checksum{}, err - } - checksum = entities.Checksum{Md5: checksums[MD5], Sha1: checksums[SHA1], Sha256: checksums[SHA256]} - return -}