From 59ae920e8ac2eaf70482ea60a310ed828c00a4b8 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 14:16:13 +0200 Subject: [PATCH] move builder_utils under arduino/builder/utils pkg --- arduino/builder/utils/utils.go | 405 ++++++++++++++++- arduino/builder/utils/utils_test.go | 5 +- legacy/builder/builder_utils/utils.go | 412 ------------------ legacy/builder/create_cmake_rule.go | 3 +- legacy/builder/phases/core_builder.go | 15 +- legacy/builder/phases/libraries_builder.go | 12 +- legacy/builder/phases/linker.go | 5 +- legacy/builder/phases/sizer.go | 5 +- legacy/builder/phases/sketch_builder.go | 6 +- legacy/builder/recipe_runner.go | 3 +- ...out_build_path_if_build_options_changed.go | 6 +- 11 files changed, 422 insertions(+), 455 deletions(-) delete mode 100644 legacy/builder/builder_utils/utils.go diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index 32e5c84435b..d9455385d76 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -2,14 +2,24 @@ package utils import ( "bytes" + "fmt" "io" "os" + "path/filepath" + "runtime" "strings" + "sync" "unicode" + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" + "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/text/runes" @@ -17,6 +27,8 @@ import ( "golang.org/x/text/unicode/norm" ) +var tr = i18n.Tr + // ObjFileIsUpToDate fixdoc func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) @@ -140,13 +152,13 @@ func NormalizeUTF8(buf []byte) []byte { var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { +// filterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func filterOutSCCS(file *paths.Path) bool { return !sourceControlFolders[file.Base()] } -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { +// filterReadableFiles is a ReadDirFilter that accepts only readable files +func filterReadableFiles(file *paths.Path) bool { // See if the file is readable by opening it f, err := file.Open() if err != nil { @@ -163,9 +175,9 @@ var filterOutHiddenFiles = paths.FilterOutPrefixes(".") func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, paths.FilterOutDirectories(), - FilterReadableFiles, + filterReadableFiles, ) if len(extensions) > 0 { fileFilter = paths.AndFilter( @@ -176,7 +188,7 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat if recurse { dirFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, ) return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) } @@ -204,10 +216,11 @@ func printableArgument(arg string) string { // This adds basic escaping which is sufficient for debug output, but // probably not for shell interpretation. This essentially reverses // ParseCommandLine. -func PrintableCommand(parts []string) string { +func printableCommand(parts []string) string { return strings.Join(f.Map(parts, printableArgument), " ") } +// ExecCommand fixdoc func ExecCommand( verbose bool, stdoutWriter, stderrWriter io.Writer, @@ -215,7 +228,7 @@ func ExecCommand( ) ([]byte, []byte, []byte, error) { verboseInfoBuf := &bytes.Buffer{} if verbose { - verboseInfoBuf.WriteString(PrintableCommand(command.GetArgs())) + verboseInfoBuf.WriteString(printableCommand(command.GetArgs())) } stdoutBuffer := &bytes.Buffer{} @@ -248,3 +261,377 @@ func ExecCommand( err = command.Wait() return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } + +// DirContentIsOlderThan DirContentIsOlderThan returns true if the content of the given directory is +// older than target file. If extensions are given, only the files with these +// extensions are tested. +func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { + targetStat, err := target.Stat() + if err != nil { + return false, err + } + targetModTime := targetStat.ModTime() + + files, err := FindFilesInFolder(dir, true, extensions...) + if err != nil { + return false, err + } + for _, file := range files { + file, err := file.Stat() + if err != nil { + return false, err + } + if file.ModTime().After(targetModTime) { + return false, nil + } + } + return true, nil +} + +// PrepareCommandForRecipe fixdoc +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { + pattern := buildProperties.Get(recipe) + if pattern == "" { + return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) + } + + commandLine := buildProperties.ExpandPropsInString(pattern) + if removeUnsetProperties { + commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) + } + + parts, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, errors.WithStack(err) + } + + // if the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + var relativePath string + if len(commandLine) > 30000 { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { + if _, err := os.Stat(arg); os.IsNotExist(err) { + continue + } + rel, err := filepath.Rel(relativePath, arg) + if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { + parts[i] = rel + } + } + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) + } + + return command, nil +} + +// CompileFiles fixdoc +func CompileFiles( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + false, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +// CompileFilesRecursive fixdoc +func CompileFilesRecursive( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + true, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +func compileFiles( + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + sourceDir *paths.Path, + recurse bool, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, + progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + validExtensions := []string{} + for ext := range globals.SourceFilesValidExtensions { + validExtensions = append(validExtensions, ext) + } + + sources, err := FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + + progress.AddSubSteps(len(sources)) + defer progress.RemoveSubSteps() + + objectFiles := paths.NewPathList() + var objectFilesMux sync.Mutex + if len(sources) == 0 { + return objectFiles, nil + } + var errorsList []error + var errorsMux sync.Mutex + + queue := make(chan *paths.Path) + job := func(source *paths.Path) { + recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) + if !buildProperties.ContainsKey(recipe) { + recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) + } + objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( + stdoutWriter, stderrWriter, + warningsLevel, + compilationDatabase, + verbose, + onlyUpdateCompilationDatabase, + sourceDir, source, buildPath, buildProperties, includes, recipe, + ) + if verbose { + verboseStdoutFn(verboseStdout) + verboseInfoFn(string(verboseInfo)) + } + verboseStderrFn(stderr) + if err != nil { + errorsMux.Lock() + errorsList = append(errorsList, err) + errorsMux.Unlock() + } else { + objectFilesMux.Lock() + objectFiles.Add(objectFile) + objectFilesMux.Unlock() + } + } + + // Spawn jobs runners + var wg sync.WaitGroup + if jobs == 0 { + jobs = runtime.NumCPU() + } + for i := 0; i < jobs; i++ { + wg.Add(1) + go func() { + for source := range queue { + job(source) + } + wg.Done() + }() + } + + // Feed jobs until error or done + for _, source := range sources { + errorsMux.Lock() + gotError := len(errorsList) > 0 + errorsMux.Unlock() + if gotError { + break + } + queue <- source + + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } + } + close(queue) + wg.Wait() + if len(errorsList) > 0 { + // output the first error + return nil, errors.WithStack(errorsList[0]) + } + objectFiles.Sort() + return objectFiles, nil +} + +func compileFileWithRecipe( + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, + compilationDatabase *builder.CompilationDatabase, + verbose, onlyUpdateCompilationDatabase bool, + sourcePath *paths.Path, + source *paths.Path, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + recipe string, +) (*paths.Path, []byte, []byte, []byte, error) { + verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) + properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) + properties.SetPath("source_file", source) + relativeSource, err := sourcePath.RelTo(source) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + depsFile := buildPath.Join(relativeSource.String() + ".d") + objectFile := buildPath.Join(relativeSource.String() + ".o") + + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + err = objectFile.Parent().MkdirAll() + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + command, err := PrepareCommandForRecipe(properties, recipe, false) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + if compilationDatabase != nil { + compilationDatabase.Add(source, command) + } + if !objIsUpToDate && !onlyUpdateCompilationDatabase { + // Since this compile could be multithreaded, we first capture the command output + info, stdout, stderr, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, Capture, Capture) + // and transfer all at once at the end... + if verbose { + verboseInfo.Write(info) + verboseStdout.Write(stdout) + } + errOut.Write(stderr) + + // ...and then return the error + if err != nil { + return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) + } + } else if verbose { + if objIsUpToDate { + verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) + } else { + verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) + } + } + + return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil +} + +// ArchiveCompiledFiles fixdoc +func ArchiveCompiledFiles( + buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, + onlyUpdateCompilationDatabase, verbose bool, + stdoutWriter, stderrWriter io.Writer, +) (*paths.Path, []byte, error) { + verboseInfobuf := &bytes.Buffer{} + archiveFilePath := buildPath.JoinPath(archiveFile) + + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + + if archiveFileStat, err := archiveFilePath.Stat(); err == nil { + rebuildArchive := false + for _, objectFile := range objectFilesToArchive { + objectFileStat, err := objectFile.Stat() + if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { + // need to rebuild the archive + rebuildArchive = true + break + } + } + + // something changed, rebuild the core archive + if rebuildArchive { + if err := archiveFilePath.Remove(); err != nil { + return nil, nil, errors.WithStack(err) + } + } else { + if verbose { + verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + } + + for _, objectFile := range objectFilesToArchive { + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) + properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + + command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + + verboseInfo, _, _, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, ShowIfVerbose /* stdout */, Show /* stderr */) + if verbose { + verboseInfobuf.WriteString(string(verboseInfo)) + } + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + } + + return archiveFilePath, verboseInfobuf.Bytes(), nil +} diff --git a/arduino/builder/utils/utils_test.go b/arduino/builder/utils/utils_test.go index c7116bdcc21..6fd54a69c90 100644 --- a/arduino/builder/utils/utils_test.go +++ b/arduino/builder/utils/utils_test.go @@ -1,9 +1,8 @@ -package utils_test +package utils import ( "testing" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/stretchr/testify/require" ) @@ -22,6 +21,6 @@ func TestPrintableCommand(t *testing.T) { " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + " lastarg" - result := utils.PrintableCommand(parts) + result := PrintableCommand(parts) require.Equal(t, correct, result) } diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go deleted file mode 100644 index 0b4c47d5d96..00000000000 --- a/legacy/builder/builder_utils/utils.go +++ /dev/null @@ -1,412 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder_utils - -import ( - "bytes" - "fmt" - "io" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - - "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/arduino/builder/progress" - "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/i18n" - rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" - "github.com/arduino/go-paths-helper" - "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" -) - -var tr = i18n.Tr - -// DirContentIsOlderThan returns true if the content of the given directory is -// older than target file. If extensions are given, only the files with these -// extensions are tested. -func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { - targetStat, err := target.Stat() - if err != nil { - return false, err - } - targetModTime := targetStat.ModTime() - - files, err := utils.FindFilesInFolder(dir, true, extensions...) - if err != nil { - return false, err - } - for _, file := range files { - file, err := file.Stat() - if err != nil { - return false, err - } - if file.ModTime().After(targetModTime) { - return false, nil - } - } - return true, nil -} - -func CompileFiles( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - false, - buildPath, buildProperties, includes, - verbose, - warningsLevel, - stdoutWriter, stderrWriter, - verboseInfoFn, verboseStdoutFn, verboseStderrFn, - progress, progressCB, - ) -} - -func CompileFilesRecursive( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - true, - buildPath, buildProperties, includes, - verbose, - warningsLevel, - stdoutWriter, stderrWriter, - verboseInfoFn, verboseStdoutFn, verboseStderrFn, - progress, progressCB, - ) -} - -func compileFiles( - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - sourceDir *paths.Path, - recurse bool, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, - progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - validExtensions := []string{} - for ext := range globals.SourceFilesValidExtensions { - validExtensions = append(validExtensions, ext) - } - - sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) - if err != nil { - return nil, err - } - - progress.AddSubSteps(len(sources)) - defer progress.RemoveSubSteps() - - objectFiles := paths.NewPathList() - var objectFilesMux sync.Mutex - if len(sources) == 0 { - return objectFiles, nil - } - var errorsList []error - var errorsMux sync.Mutex - - queue := make(chan *paths.Path) - job := func(source *paths.Path) { - recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) - if !buildProperties.ContainsKey(recipe) { - recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) - } - objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( - stdoutWriter, stderrWriter, - warningsLevel, - compilationDatabase, - verbose, - onlyUpdateCompilationDatabase, - sourceDir, source, buildPath, buildProperties, includes, recipe, - ) - if verbose { - verboseStdoutFn(verboseStdout) - verboseInfoFn(string(verboseInfo)) - } - verboseStderrFn(stderr) - if err != nil { - errorsMux.Lock() - errorsList = append(errorsList, err) - errorsMux.Unlock() - } else { - objectFilesMux.Lock() - objectFiles.Add(objectFile) - objectFilesMux.Unlock() - } - } - - // Spawn jobs runners - var wg sync.WaitGroup - if jobs == 0 { - jobs = runtime.NumCPU() - } - for i := 0; i < jobs; i++ { - wg.Add(1) - go func() { - for source := range queue { - job(source) - } - wg.Done() - }() - } - - // Feed jobs until error or done - for _, source := range sources { - errorsMux.Lock() - gotError := len(errorsList) > 0 - errorsMux.Unlock() - if gotError { - break - } - queue <- source - - progress.CompleteStep() - // PushProgress - if progressCB != nil { - progressCB(&rpc.TaskProgress{ - Percent: progress.Progress, - Completed: progress.Progress >= 100.0, - }) - } - } - close(queue) - wg.Wait() - if len(errorsList) > 0 { - // output the first error - return nil, errors.WithStack(errorsList[0]) - } - objectFiles.Sort() - return objectFiles, nil -} - -func compileFileWithRecipe( - stdoutWriter, stderrWriter io.Writer, - warningsLevel string, - compilationDatabase *builder.CompilationDatabase, - verbose, onlyUpdateCompilationDatabase bool, - sourcePath *paths.Path, - source *paths.Path, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - recipe string, -) (*paths.Path, []byte, []byte, []byte, error) { - verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} - - properties := buildProperties.Clone() - properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) - properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) - properties.SetPath("source_file", source) - relativeSource, err := sourcePath.RelTo(source) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - depsFile := buildPath.Join(relativeSource.String() + ".d") - objectFile := buildPath.Join(relativeSource.String() + ".o") - - properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) - err = objectFile.Parent().MkdirAll() - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - command, err := PrepareCommandForRecipe(properties, recipe, false) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - if compilationDatabase != nil { - compilationDatabase.Add(source, command) - } - if !objIsUpToDate && !onlyUpdateCompilationDatabase { - // Since this compile could be multithreaded, we first capture the command output - info, stdout, stderr, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture, utils.Capture) - // and transfer all at once at the end... - if verbose { - verboseInfo.Write(info) - verboseStdout.Write(stdout) - } - errOut.Write(stderr) - - // ...and then return the error - if err != nil { - return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) - } - } else if verbose { - if objIsUpToDate { - verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) - } else { - verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) - } - } - - return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil -} - -func ArchiveCompiledFiles( - buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, - onlyUpdateCompilationDatabase, verbose bool, - stdoutWriter, stderrWriter io.Writer, -) (*paths.Path, []byte, error) { - verboseInfobuf := &bytes.Buffer{} - archiveFilePath := buildPath.JoinPath(archiveFile) - - if onlyUpdateCompilationDatabase { - if verbose { - verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - - if archiveFileStat, err := archiveFilePath.Stat(); err == nil { - rebuildArchive := false - for _, objectFile := range objectFilesToArchive { - objectFileStat, err := objectFile.Stat() - if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { - // need to rebuild the archive - rebuildArchive = true - break - } - } - - // something changed, rebuild the core archive - if rebuildArchive { - if err := archiveFilePath.Remove(); err != nil { - return nil, nil, errors.WithStack(err) - } - } else { - if verbose { - verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - } - - for _, objectFile := range objectFilesToArchive { - properties := buildProperties.Clone() - properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) - properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) - properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) - - command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - - verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if verbose { - verboseInfobuf.WriteString(string(verboseInfo)) - } - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - } - - return archiveFilePath, verboseInfobuf.Bytes(), nil -} - -const COMMANDLINE_LIMIT = 30000 - -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { - pattern := buildProperties.Get(recipe) - if pattern == "" { - return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) - } - - commandLine := buildProperties.ExpandPropsInString(pattern) - if removeUnsetProperties { - commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) - } - - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, errors.WithStack(err) - } - - // if the overall commandline is too long for the platform - // try reducing the length by making the filenames relative - // and changing working directory to build.path - var relativePath string - if len(commandLine) > COMMANDLINE_LIMIT { - relativePath = buildProperties.Get("build.path") - for i, arg := range parts { - if _, err := os.Stat(arg); os.IsNotExist(err) { - continue - } - rel, err := filepath.Rel(relativePath, arg) - if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - parts[i] = rel - } - } - } - - command, err := executils.NewProcess(nil, parts...) - if err != nil { - return nil, errors.WithStack(err) - } - if relativePath != "" { - command.SetDir(relativePath) - } - - return command, nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 6d6b67ae1d6..9ce4b068832 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -28,7 +28,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" ) @@ -367,7 +366,7 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) + command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index b6ced10ad86..36a8335d1bf 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -26,11 +26,11 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" @@ -38,9 +38,6 @@ import ( "github.com/pkg/errors" ) -// Trasformo in un command -// provo a spostare builder_utils in arduino/Builder - var tr = i18n.Tr func CoreBuilder( @@ -116,7 +113,7 @@ func compileCore( var err error variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { - variantObjectFiles, err = builder_utils.CompileFilesRecursive( + variantObjectFiles, err = utils.CompileFilesRecursive( variantFolder, buildPath, buildProperties, includes, onlyUpdateCompilationDatabase, compilationDatabase, @@ -149,12 +146,12 @@ func compileCore( var canUseArchivedCore bool if onlyUpdateCompilationDatabase || clean { canUseArchivedCore = false - } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the core files (including platform.txt) has changed canUseArchivedCore = false } else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) { canUseArchivedCore = true - } else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the build core files (including platform.txt) has changed canUseArchivedCore = false } else { @@ -170,7 +167,7 @@ func compileCore( } } - coreObjectFiles, err := builder_utils.CompileFilesRecursive( + coreObjectFiles, err := utils.CompileFilesRecursive( coreFolder, buildPath, buildProperties, includes, onlyUpdateCompilationDatabase, compilationDatabase, @@ -185,7 +182,7 @@ func compileCore( return nil, nil, errors.WithStack(err) } - archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( buildPath, paths.New("core.a"), coreObjectFiles, buildProperties, onlyUpdateCompilationDatabase, verbose, stdoutWriter, stderrWriter, ) diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index bbd02bada1a..15d34383989 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -19,9 +19,9 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -67,7 +67,7 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) + command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { @@ -186,7 +186,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } if library.Layout == libraries.RecursiveLayout { - libObjectFiles, err := builder_utils.CompileFilesRecursive( + libObjectFiles, err := utils.CompileFilesRecursive( library.SourceDir, libraryBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -203,7 +203,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p return nil, errors.WithStack(err) } if library.DotALinkage { - archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties, ctx.OnlyUpdateCompilationDatabase, ctx.Verbose, ctx.Stdout, ctx.Stderr, @@ -222,7 +222,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } - libObjectFiles, err := builder_utils.CompileFiles( + libObjectFiles, err := utils.CompileFiles( library.SourceDir, libraryBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -242,7 +242,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { utilityBuildPath := libraryBuildPath.Join("utility") - utilityObjectFiles, err := builder_utils.CompileFiles( + utilityObjectFiles, err := utils.CompileFiles( library.UtilityDir, utilityBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 8d01d55b4b4..ed1d8399a26 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -94,7 +93,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) + command, err := utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { return errors.WithStack(err) } @@ -118,7 +117,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) + command, err := utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err } diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index a4c89a4bc84..b0261daa105 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -22,7 +22,6 @@ import ( "strconv" "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -50,7 +49,7 @@ func (s *Sizer) Run(ctx *types.Context) error { } func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -182,7 +181,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index a6e4ad14314..bd71c06cd63 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -17,8 +17,8 @@ package phases import ( "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" ) @@ -35,7 +35,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { return errors.WithStack(err) } - objectFiles, err := builder_utils.CompileFiles( + objectFiles, err := utils.CompileFiles( sketchBuildPath, sketchBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -55,7 +55,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := sketchBuildPath.Join("src") if sketchSrcPath.IsDir() { - srcObjectFiles, err := builder_utils.CompileFilesRecursive( + srcObjectFiles, err := utils.CompileFilesRecursive( sketchSrcPath, sketchSrcPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index c5663705230..9e8bdcd2425 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -20,7 +20,6 @@ import ( "sort" "strings" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/arduino/builder/utils" properties "github.com/arduino/go-properties-orderedmap" @@ -44,7 +43,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) + command, err := utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/wipeout_build_path_if_build_options_changed.go b/legacy/builder/wipeout_build_path_if_build_options_changed.go index 2a6873de97b..72b00b8671a 100644 --- a/legacy/builder/wipeout_build_path_if_build_options_changed.go +++ b/legacy/builder/wipeout_build_path_if_build_options_changed.go @@ -19,7 +19,7 @@ import ( "encoding/json" "path/filepath" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -66,9 +66,9 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties.GetPath("build.core.path") realCoreFolder := coreFolder.Parent().Parent() jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE) - coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") + coreUnchanged, _ := utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) { - coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") + coreUnchanged, _ = utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") } if coreUnchanged { return nil