diff --git a/cmd/cmd.go b/cmd/cmd.go index 100ee6b..11914a5 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -17,6 +17,7 @@ var protocolRegExp *gofrogcmd.CmdOutputPattern var unrecognizedImportRegExp *gofrogcmd.CmdOutputPattern var notFoundRegExp *gofrogcmd.CmdOutputPattern var unknownRevisionRegExp *gofrogcmd.CmdOutputPattern +var notFoundZipRegExp *gofrogcmd.CmdOutputPattern func NewCmd() (*Cmd, error) { execPath, err := exec.LookPath("go") @@ -70,11 +71,11 @@ func RunGo(goArg []string) error { return err } goCmd.Command = goArg - err = prepareCmdOutputPattern() + err = prepareRegExp() if err != nil { return err } - _, _, err = gofrogcmd.RunCmdWithOutputParser(goCmd, true, protocolRegExp, notFoundRegExp, unrecognizedImportRegExp, unknownRevisionRegExp) + _, _, err = gofrogcmd.RunCmdWithOutputParser(goCmd, true, protocolRegExp, notFoundRegExp, unrecognizedImportRegExp, unknownRevisionRegExp, notFoundZipRegExp) return errorutils.CheckError(err) } @@ -119,7 +120,7 @@ func GetDependenciesGraph() (map[string]bool, error) { } goCmd.Command = []string{"mod", "graph"} - err = prepareCmdOutputPattern() + err = prepareGlobalRegExp() if err != nil { return nil, err } diff --git a/cmd/utils.go b/cmd/utils.go index 8f02c7a..05dd4a3 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -14,8 +14,16 @@ import ( "strings" ) +func prepareRegExp() error { + err := prepareGlobalRegExp() + if err != nil { + return err + } + return prepareNotFoundZipRegExp() +} + // Compiles all the regex once -func prepareCmdOutputPattern() error { +func prepareGlobalRegExp() error { var err error if protocolRegExp == nil { log.Debug("Initializing protocol regexp") @@ -24,32 +32,38 @@ func prepareCmdOutputPattern() error { return err } } - if notFoundRegExp == nil || unrecognizedImportRegExp == nil || unknownRevisionRegExp == nil { - if notFoundRegExp == nil { - log.Debug("Initializing not found regexp") - notFoundRegExp, err = initRegExp(`^go: ([^\/\r\n]+\/[^\r\n\s:]*).*(404( Not Found)?[\s]?)$`, Error) - if err != nil { - return err - } - } - if unrecognizedImportRegExp == nil { - log.Debug("Initializing unrecognized import path regexp") - unrecognizedImportRegExp, err = initRegExp(`[^go:]([^\/\r\n]+\/[^\r\n\s:]*).*(unrecognized import path)`, Error) - if err != nil { - return err - } + if notFoundRegExp == nil { + log.Debug("Initializing not found regexp") + notFoundRegExp, err = initRegExp(`^go: ([^\/\r\n]+\/[^\r\n\s:]*).*(404( Not Found)?[\s]?)$`, Error) + if err != nil { + return err } + } - if unknownRevisionRegExp == nil { - log.Debug("Initializing unknown revision regexp") - unknownRevisionRegExp, err = initRegExp(`[^go:]([^\/\r\n]+\/[^\r\n\s:]*).*(unknown revision)`, Error) - if err != nil { - return err - } + if unrecognizedImportRegExp == nil { + log.Debug("Initializing unrecognized import path regexp") + unrecognizedImportRegExp, err = initRegExp(`[^go:]([^\/\r\n]+\/[^\r\n\s:]*).*(unrecognized import path)`, Error) + if err != nil { + return err } } - return nil + + if unknownRevisionRegExp == nil { + log.Debug("Initializing unknown revision regexp") + unknownRevisionRegExp, err = initRegExp(`[^go:]([^\/\r\n]+\/[^\r\n\s:]*).*(unknown revision)`, Error) + } + + return err +} + +func prepareNotFoundZipRegExp() error { + var err error + if notFoundZipRegExp == nil { + log.Debug("Initializing not found zip file") + notFoundZipRegExp, err = initRegExp(`unknown import path ["]([^\/\r\n]+\/[^\r\n\s:]*)["].*(404( Not Found)?[\s]?)$`, Error) + } + return err } func initRegExp(regex string, execFunc func(pattern *gofrogio.CmdOutputPattern) (string, error)) (*gofrogio.CmdOutputPattern, error) { @@ -72,7 +86,10 @@ func MaskCredentials(pattern *gofrogio.CmdOutputPattern) (string, error) { } func Error(pattern *gofrogio.CmdOutputPattern) (string, error) { - fmt.Fprintf(os.Stderr, pattern.Line) + _, err := fmt.Fprint(os.Stderr, pattern.Line) + if err != nil { + return "", errorutils.CheckError(err) + } if len(pattern.MatchedResults) >= 3 { return "", errors.New(pattern.MatchedResults[2] + ":" + strings.TrimSpace(pattern.MatchedResults[1])) } diff --git a/executers/dependenciesutils.go b/executers/dependenciesutils.go index 9f64d70..e3bd71c 100644 --- a/executers/dependenciesutils.go +++ b/executers/dependenciesutils.go @@ -111,11 +111,11 @@ func downloadDependencies(targetRepo string, cache *cache.DependenciesCache, dep } if resp.StatusCode == 200 { - cacheDependenciesMap[getDependencyName(nameAndVersion[0])+":"+nameAndVersion[1]] = true + cacheDependenciesMap[goModEncode(nameAndVersion[0])+":"+goModEncode(nameAndVersion[1])] = true err = downloadDependency(true, module, targetRepo, auth) dependenciesMap[module] = true } else if resp.StatusCode == 404 { - cacheDependenciesMap[getDependencyName(nameAndVersion[0])+":"+nameAndVersion[1]] = false + cacheDependenciesMap[goModEncode(nameAndVersion[0])+":"+goModEncode(nameAndVersion[1])] = false err = downloadDependency(false, module, targetRepo, nil) dependenciesMap[module] = false } @@ -154,17 +154,35 @@ func createDependencyInTemp(zipPath string) (tempDir string, err error) { return tempDir, nil } -func replaceExclamationMarkWithUpperCase(moduleName string) string { +// Returns the actual path to the dependency. +// If in the path there are capital letters, the Go convention is to use "!" before the letter. +// The letter itself in lowercase. +func goModEncode(name string) string { + path := "" + for _, letter := range name { + if unicode.IsUpper(letter) { + path += "!" + strings.ToLower(string(letter)) + } else { + path += string(letter) + } + } + return path +} + +// Returns the path to the dependency decoded form lower case to upper case +// If in the path there are capital letters, the Go convention is to use "!" before the letter. +// The letter itself in lowercase. This function will decode back to Upper case +func goModDecode(name string) string { var str string - for i := 0; i < len(moduleName); i++ { - if string(moduleName[i]) == "!" { - if i < len(moduleName)-1 { - r := rune(moduleName[i+1]) + for i := 0; i < len(name); i++ { + if string(name[i]) == "!" { + if i < len(name)-1 { + r := rune(name[i+1]) str += string(unicode.ToUpper(r)) i++ } } else { - str += string(moduleName[i]) + str += string(name[i]) } } return str @@ -248,8 +266,8 @@ func GetDependencies(cachePath string, moduleSlice map[string]bool) ([]Package, var deps []Package for module := range moduleSlice { moduleInfo := strings.Split(module, "@") - name := getDependencyName(moduleInfo[0]) - dep, err := createDependency(cachePath, name, moduleInfo[1]) + name := goModEncode(moduleInfo[0]) + dep, err := createDependency(cachePath, name, goModEncode(moduleInfo[1])) if err != nil { return nil, err } @@ -260,21 +278,6 @@ func GetDependencies(cachePath string, moduleSlice map[string]bool) ([]Package, return deps, nil } -// Returns the actual path to the dependency. -// If in the path there are capital letters, the Go convention is to use "!" before the letter. -// The letter itself in lowercase. -func getDependencyName(name string) string { - path := "" - for _, letter := range name { - if unicode.IsUpper(letter) { - path += "!" + strings.ToLower(string(letter)) - } else { - path += string(letter) - } - } - return path -} - // Creates a go dependency. // Returns a nil value in case the dependency does not include a zip in the cache. func createDependency(cachePath, dependencyName, version string) (*Package, error) { @@ -385,10 +388,6 @@ func mergeReplaceDependenciesWithGraphDependencies(replaceDeps []string, graphDe } func getReplaceDependencies() ([]string, error) { - replaceRegExp, err := clientutils.GetRegExp(`\s*replace (?:[\(\w\.@:%_\+-.~#?&]?.+)`) - if err != nil { - return nil, err - } rootDir, err := cmd.GetProjectRoot() if err != nil { return nil, err @@ -398,8 +397,35 @@ func getReplaceDependencies() ([]string, error) { if err != nil { return nil, err } - replaceDependencies := replaceRegExp.FindAllString(string(modFileContent), -1) - return replaceDependencies, nil + return parseModForReplaceDependencies(string(modFileContent)) +} + +func parseModForReplaceDependencies(modFileContent string) ([]string, error) { + replaceLinerRegExp, err := clientutils.GetRegExp(`[^\s*]?replace (?:[\(\w\.@:%_\+-.~#?&]?.+)=>(?:[\(\w\.@:%_\+-.~#?&]?.+)`) + if err != nil { + return nil, err + } + replaceLinerDependencies := replaceLinerRegExp.FindAllString(modFileContent, -1) + replaceRegExp, err := clientutils.GetRegExp(`\s*replace\s*\(`) + if err != nil { + return replaceLinerDependencies, err + } + replaceDependencies := replaceRegExp.FindAllString(modFileContent, -1) + if len(replaceDependencies) > 0 { + log.Debug("Found replace block...") + replacePosition := strings.Index(modFileContent, replaceDependencies[0]) + lines := strings.Split(modFileContent[replacePosition+len(replaceDependencies[0]):], "\n") + for _, line := range lines { + if line == ")" { + break + } + if line == "" || line == "\n" { + continue + } + replaceLinerDependencies = append(replaceLinerDependencies, line) + } + } + return replaceLinerDependencies, nil } // Runs go mod graph command with fallback. @@ -407,7 +433,7 @@ func getDependenciesGraphWithFallback(targetRepo string, auth auth.ArtifactoryDe dependenciesMap := map[string]bool{} modulesWithErrors := map[string]previousTries{} usedProxy := true - for true { + for { // Configuring each run to use Artifactory/VCS err := setOrUnsetGoProxy(usedProxy, targetRepo, auth) if err != nil { diff --git a/executers/dependenciesutils_test.go b/executers/dependenciesutils_test.go index b8ee33a..8f3d847 100644 --- a/executers/dependenciesutils_test.go +++ b/executers/dependenciesutils_test.go @@ -3,6 +3,7 @@ package executers import ( "fmt" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "io/ioutil" "os" "path/filepath" "reflect" @@ -39,7 +40,7 @@ func TestGetPackageZipLocation(t *testing.T) { } } -func TestGetDependencyName(t *testing.T) { +func TestEncodeDecodePath(t *testing.T) { tests := []struct { dependencyName string expectedPath string @@ -53,9 +54,13 @@ func TestGetDependencyName(t *testing.T) { for _, test := range tests { t.Run(test.dependencyName, func(t *testing.T) { - actual := getDependencyName(test.dependencyName) - if test.expectedPath != actual { - t.Errorf("Test name: %s: Expected: %s, Got: %s", test.dependencyName, test.expectedPath, actual) + encoded := goModEncode(test.dependencyName) + if test.expectedPath != encoded { + t.Errorf("Test name: %s: Expected: %s, Got: %s", test.dependencyName, test.expectedPath, encoded) + } + decoded := goModDecode(test.expectedPath) + if test.dependencyName != decoded { + t.Errorf("Test name: %s: Expected: %s, Got: %s", test.dependencyName, test.dependencyName, decoded) } }) } @@ -145,6 +150,8 @@ func TestMergeReplaceDependenciesWithGraphDependencies(t *testing.T) { map[string]bool{"github.com/jfrog/jfrog-client-go@v0.1.0": true}}, {"addToGraphMap", []string{"replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v0.1.0", "replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-cli-go", "replace github.com/jfrog/jfrog-client-go => /path/to/mod/file"}, map[string]bool{"github.com/jfrog/jfrog-cli-go@v1.21.0": true}, map[string]bool{"github.com/jfrog/jfrog-cli-go@v1.21.0": true, "github.com/jfrog/jfrog-client-go@v0.1.0": true}}, + {"addToGraphMapFromReplaceBlock", []string{"replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v0.1.0", "replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-cli-go", "replace github.com/jfrog/jfrog-client-go => /path/to/mod/file", "github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v2.1.2"}, map[string]bool{"github.com/jfrog/jfrog-cli-go@v1.21.0": true}, + map[string]bool{"github.com/jfrog/jfrog-cli-go@v1.21.0": true, "github.com/jfrog/jfrog-client-go@v0.1.0": true, "github.com/jfrog/jfrog-client-go@v2.1.2": true}}, } for _, test := range tests { @@ -157,6 +164,47 @@ func TestMergeReplaceDependenciesWithGraphDependencies(t *testing.T) { } } +func TestParseModForReplaceDependencies(t *testing.T) { + testdata, err := getBaseDir() + if err != nil { + t.Error(err) + } + + modDir := testdata + fileutils.GetFileSeparator() + "mods" + fileutils.GetFileSeparator() + tests := []struct { + name string + expectedReplaceDependencies []string + }{ + {"replaceBlockFirst", []string{" github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", " github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + { + "replaceBlockLast", []string{" github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", " github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + { + "replaceLineFirst", []string{"replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", "replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + { + "replaceLineLast", []string{"replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", "replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + { + "replaceBothLineFirst", []string{"replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", "replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1", " github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", " github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + { + "replaceBothBlockFirst", []string{"replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", "replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1", " github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible", " github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + modContent, err := ioutil.ReadFile(modDir + test.name + ".txt") + if err != nil { + t.Error(err) + } + replaceLinerDependencies, err := parseModForReplaceDependencies(string(modContent)) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(test.expectedReplaceDependencies, replaceLinerDependencies) { + t.Errorf("Test name: %s: Expected: %v, Got: %v", test.name, test.expectedReplaceDependencies, replaceLinerDependencies) + } + }) + } +} + func getBaseDir() (baseDir string, err error) { pwd, err := os.Getwd() if err != nil { diff --git a/executers/populateandpublish.go b/executers/populateandpublish.go index 7f91786..ea557f5 100644 --- a/executers/populateandpublish.go +++ b/executers/populateandpublish.go @@ -256,7 +256,7 @@ func (pwd *PackageWithDeps) prepareAndRunInit(pathToModFile string) error { // If empty, run go mod init moduleId := pwd.Dependency.GetId() moduleInfo := strings.Split(moduleId, ":") - return cmd.RunGoModInit(replaceExclamationMarkWithUpperCase(moduleInfo[0])) + return cmd.RunGoModInit(goModDecode(moduleInfo[0])) } func writeModContentToModFile(path string, modContent []byte) error { @@ -266,7 +266,10 @@ func writeModContentToModFile(path string, modContent []byte) error { func (pwd *PackageWithDeps) getModPathInTemp(tempDir string) string { moduleId := pwd.Dependency.GetId() moduleInfo := strings.Split(moduleId, ":") - moduleInfo[0] = replaceExclamationMarkWithUpperCase(moduleInfo[0]) + moduleInfo[0] = goModDecode(moduleInfo[0]) + if len(moduleInfo) > 1 { + moduleInfo[1] = goModDecode(moduleInfo[1]) + } moduleId = strings.Join(moduleInfo, ":") modulePath := strings.Replace(moduleId, ":", "@", 1) path := filepath.Join(tempDir, modulePath, "go.mod") @@ -325,11 +328,12 @@ func (pwd *PackageWithDeps) setTransitiveDependencies(targetRepo string, graphDe module := strings.Split(transitiveDependency, "@") if len(module) == 2 { dependenciesMap := cache.GetMap() - name := getDependencyName(module[0]) - _, exists := dependenciesMap[name+":"+module[1]] + name := goModEncode(module[0]) + version := goModEncode(module[1]) + _, exists := dependenciesMap[name+":"+version] if !exists { // Check if the dependency is in the local cache. - dep, err := createDependency(pwd.cachePath, name, module[1]) + dep, err := createDependency(pwd.cachePath, name, version) utils.LogError(err) if err != nil { continue @@ -346,7 +350,7 @@ func (pwd *PackageWithDeps) setTransitiveDependencies(targetRepo string, graphDe } if dep == nil { // Dependency is missing in the local cache. Need to download it... - dep, err = downloadAndCreateDependency(pwd.cachePath, name, module[1], transitiveDependency, targetRepo, downloadedFromArtifactory, auth) + dep, err = downloadAndCreateDependency(pwd.cachePath, name, version, transitiveDependency, targetRepo, downloadedFromArtifactory, auth) utils.LogError(err) if err != nil { continue @@ -360,7 +364,7 @@ func (pwd *PackageWithDeps) setTransitiveDependencies(targetRepo string, graphDe cachePath: pwd.cachePath, GoModEditMessage: pwd.GoModEditMessage} dependencies = append(dependencies, *depsWithTrans) - dependenciesMap[name+":"+module[1]] = downloadedFromArtifactory + dependenciesMap[name+":"+version] = downloadedFromArtifactory } } else { log.Debug("Dependency", transitiveDependency, "has been previously added.") diff --git a/testdata/mods/replaceBlockFirst.txt b/testdata/mods/replaceBlockFirst.txt new file mode 100644 index 0000000..15ac33c --- /dev/null +++ b/testdata/mods/replaceBlockFirst.txt @@ -0,0 +1,11 @@ +module jfrog.com/jfrog-router + +replace ( + github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible + github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 +) + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) \ No newline at end of file diff --git a/testdata/mods/replaceBlockLast.txt b/testdata/mods/replaceBlockLast.txt new file mode 100644 index 0000000..ea192a4 --- /dev/null +++ b/testdata/mods/replaceBlockLast.txt @@ -0,0 +1,11 @@ +module jfrog.com/jfrog-router + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) + +replace ( + github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible + github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 +) diff --git a/testdata/mods/replaceBothBlockFirst.txt b/testdata/mods/replaceBothBlockFirst.txt new file mode 100644 index 0000000..407d55f --- /dev/null +++ b/testdata/mods/replaceBothBlockFirst.txt @@ -0,0 +1,14 @@ +module jfrog.com/jfrog-router + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) + +replace ( + github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible + github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 +) + +replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible +replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 \ No newline at end of file diff --git a/testdata/mods/replaceBothLineFirst.txt b/testdata/mods/replaceBothLineFirst.txt new file mode 100644 index 0000000..15402cf --- /dev/null +++ b/testdata/mods/replaceBothLineFirst.txt @@ -0,0 +1,14 @@ +module jfrog.com/jfrog-router + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) + +replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible +replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 + +replace ( + github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible + github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 +) \ No newline at end of file diff --git a/testdata/mods/replaceLineFirst.txt b/testdata/mods/replaceLineFirst.txt new file mode 100644 index 0000000..bd349c9 --- /dev/null +++ b/testdata/mods/replaceLineFirst.txt @@ -0,0 +1,9 @@ +module jfrog.com/jfrog-router + +replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible +replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1 + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) \ No newline at end of file diff --git a/testdata/mods/replaceLineLast.txt b/testdata/mods/replaceLineLast.txt new file mode 100644 index 0000000..5431294 --- /dev/null +++ b/testdata/mods/replaceLineLast.txt @@ -0,0 +1,9 @@ +module jfrog.com/jfrog-router + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect + contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect +) + +replace github.com/Masterminds/sprig => github.com/Masterminds/sprig v2.13.0+incompatible +replace github.com/Microsoft/ApplicationInsights-Go => github.com/Microsoft/ApplicationInsights-Go v0.3.1