diff --git a/legacy/builder/types/accessories.go b/arduino/builder/builder.go similarity index 57% rename from legacy/builder/types/accessories.go rename to arduino/builder/builder.go index b4de7a1a18a..7408ece5c3f 100644 --- a/legacy/builder/types/accessories.go +++ b/arduino/builder/builder.go @@ -13,29 +13,18 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package types +package builder -import "golang.org/x/exp/slices" +import "github.com/arduino/arduino-cli/arduino/sketch" -type UniqueSourceFileQueue []*SourceFile - -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { - *queue = append(*queue, value) - } -} - -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { - return slices.ContainsFunc(queue, target.Equals) +// Builder is a Sketch builder. +type Builder struct { + sketch *sketch.Sketch } -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue UniqueSourceFileQueue) Empty() bool { - return len(queue) == 0 +// NewBuilder creates a sketch Builder. +func NewBuilder(sk *sketch.Sketch) *Builder { + return &Builder{ + sketch: sk, + } } diff --git a/arduino/builder/compilation_database.go b/arduino/builder/compilation_database.go index 54a91d0c707..6a7e2e475dd 100644 --- a/arduino/builder/compilation_database.go +++ b/arduino/builder/compilation_database.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" "os" - "os/exec" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" ) @@ -67,25 +67,22 @@ func (db *CompilationDatabase) SaveToFile() { } } -func dirForCommand(command *exec.Cmd) string { - // This mimics what Cmd.Run also does: Use Dir if specified, - // current directory otherwise - if command.Dir != "" { - return command.Dir - } - dir, err := os.Getwd() - if err != nil { - fmt.Println(tr("Error getting current directory for compilation database: %s", err)) - return "" +// Add adds a new CompilationDatabase entry +func (db *CompilationDatabase) Add(target *paths.Path, command *executils.Process) { + commandDir := command.GetDir() + if commandDir == "" { + // This mimics what Cmd.Run also does: Use Dir if specified, + // current directory otherwise + dir, err := os.Getwd() + if err != nil { + fmt.Println(tr("Error getting current directory for compilation database: %s", err)) + } + commandDir = dir } - return dir -} -// Add adds a new CompilationDatabase entry -func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { entry := CompilationCommand{ - Directory: dirForCommand(command), - Arguments: command.Args, + Directory: commandDir, + Arguments: command.GetArgs(), File: target.String(), } diff --git a/arduino/builder/compilation_database_test.go b/arduino/builder/compilation_database_test.go index c7f5bc3003d..8a715533617 100644 --- a/arduino/builder/compilation_database_test.go +++ b/arduino/builder/compilation_database_test.go @@ -16,9 +16,9 @@ package builder import ( - "os/exec" "testing" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -28,7 +28,8 @@ func TestCompilationDatabase(t *testing.T) { require.NoError(t, err) defer tmpfile.Remove() - cmd := exec.Command("gcc", "arg1", "arg2") + cmd, err := executils.NewProcess(nil, "gcc", "arg1", "arg2") + require.NoError(t, err) db := NewCompilationDatabase(tmpfile) db.Add(paths.New("test"), cmd) db.SaveToFile() diff --git a/arduino/builder/cpp/cpp.go b/arduino/builder/cpp/cpp.go index 71c2b696702..53123434fe1 100644 --- a/arduino/builder/cpp/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -106,3 +106,8 @@ func ParseString(line string) (string, string, bool) { i += width } } + +// WrapWithHyphenI fixdoc +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go new file mode 100644 index 00000000000..dc64b0319e5 --- /dev/null +++ b/arduino/builder/detector/detector.go @@ -0,0 +1,796 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "golang.org/x/exp/slices" + + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/arduino/libraries" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +var tr = i18n.Tr + +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + onlyUpdateCompilationDatabase bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + verboseStdoutFn func(data []byte) + verboseStderrFn func(data []byte) + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult + includeFolders paths.PathList +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + verbose, useCachedLibrariesResolution bool, + onlyUpdateCompilationDatabase bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), + verboseStdoutFn func(data []byte), + verboseStderrFn func(data []byte), +) *SketchLibrariesDetector { + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + verboseStdoutFn: verboseStdoutFn, + verboseStderrFn: verboseStderrFn, + importedLibraries: libraries.List{}, + includeFolders: paths.PathList{}, + onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) resolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +// ImportedLibraries todo +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + +// IncludeFolders fixdoc +func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { + // TODO should we do a deep copy? + return l.includeFolders +} + +// appendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +// Original comment: +// Append the given folder to the include path and match or append it to +// the cache. sourceFilePath and include indicate the source of this +// include (e.g. what #include line in what file it was resolved from) +// and should be the empty string for the default include folders, like +// the core or variant. +func (l *SketchLibrariesDetector) appendIncludeFolder( + cache *includeCache, + sourceFilePath *paths.Path, + include string, + folder *paths.Path, +) { + l.includeFolders = append(l.includeFolders, folder) + cache.ExpectEntry(sourceFilePath, include, folder) +} + +// FindIncludes todo +func (l *SketchLibrariesDetector) FindIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + err := l.findIncludes(buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch) + if err != nil && l.onlyUpdateCompilationDatabase { + l.verboseInfoFn( + fmt.Sprintf( + "%s: %s", + tr("An error occurred detecting libraries"), + tr("the compilation database may be incomplete or inaccurate"), + ), + ) + return nil + } + return err +} + +func (l *SketchLibrariesDetector) findIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + librariesResolutionCache := buildPath.Join("libraries.cache") + if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { + d, err := librariesResolutionCache.ReadFile() + if err != nil { + return err + } + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { + return err + } + if l.verbose { + l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) + } + return nil + } + + cachePath := buildPath.Join("includes.cache") + cache := readCache(cachePath) + + l.appendIncludeFolder(cache, nil, "", buildCorePath) + if buildVariantPath != nil { + l.appendIncludeFolder(cache, nil, "", buildVariantPath) + } + + sourceFileQueue := &uniqueSourceFileQueue{} + + if !l.useCachedLibrariesResolution { + sketch := sketch + mergedfile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.push(mergedfile) + + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) + srcSubfolderPath := sketchBuildPath.Join("src") + if srcSubfolderPath.IsDir() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) + } + + for !sourceFileQueue.empty() { + err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) + if err != nil { + cachePath.Remove() + return errors.WithStack(err) + } + } + + // Finalize the cache + cache.ExpectEnd() + if err := writeCache(cache, cachePath); err != nil { + return errors.WithStack(err) + } + } + + if err := l.failIfImportedLibraryIsWrong(); err != nil { + return errors.WithStack(err) + } + + if d, err := json.Marshal(l.includeFolders); err != nil { + return err + } else if err := librariesResolutionCache.WriteFile(d); err != nil { + return err + } + + return nil +} + +func (l *SketchLibrariesDetector) findIncludesUntilDone( + cache *includeCache, + sourceFileQueue *uniqueSourceFileQueue, + buildProperties *properties.Map, + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + platformArch string, +) error { + sourceFile := sourceFileQueue.pop() + sourcePath := sourceFile.SourcePath() + targetFilePath := paths.NullPath() + depPath := sourceFile.DepfilePath() + objPath := sourceFile.ObjectPath() + + // TODO: This should perhaps also compare against the + // include.cache file timestamp. Now, it only checks if the file + // changed after the object file was generated, but if it + // changed between generating the cache and the object file, + // this could show the file as unchanged when it really is + // changed. Changing files during a build isn't really + // supported, but any problems from it should at least be + // resolved when doing another build, which is not currently the + // case. + // TODO: This reads the dependency file, but the actual building + // does it again. Should the result be somehow cached? Perhaps + // remove the object file if it is found to be stale? + unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) + if err != nil { + return errors.WithStack(err) + } + + first := true + for { + cache.ExpectFile(sourcePath) + + // Libraries may require the "utility" directory to be added to the include + // search path, but only for the source code of the library, so we temporary + // copy the current search path list and add the library' utility directory + // if needed. + includeFolders := l.includeFolders + if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { + includeFolders = append(includeFolders, extraInclude) + } + + var preprocErr error + var preprocStderr []byte + + var missingIncludeH string + if unchanged && cache.valid { + missingIncludeH = cache.Next().Include + if first && l.verbose { + l.verboseInfoFn(tr("Using cached library dependencies for file: %[1]s", sourcePath)) + } + } else { + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + // Unwrap error and see if it is an ExitError. + if preprocErr == nil { + // Preprocessor successful, done + missingIncludeH = "" + } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { + // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors + return errors.WithStack(preprocErr) + } else { + missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) + if missingIncludeH == "" && l.verbose { + l.verboseInfoFn(tr("Error while detecting libraries included by %[1]s", sourcePath)) + } + } + } + + if missingIncludeH == "" { + // No missing includes found, we're done + cache.ExpectEntry(sourcePath, "", nil) + return nil + } + + library := l.resolveLibrary(missingIncludeH, platformArch) + if library == nil { + // Library could not be resolved, show error + if preprocErr == nil || preprocStderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + if preprocErr == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New(tr("Internal error in cache")) + } + } + l.verboseStderrFn(preprocStderr) + return errors.WithStack(preprocErr) + } + + // Add this library to the list of libraries, the + // include path and queue its source files for further + // include scanning + l.AppendImportedLibraries(library) + l.appendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + + if library.Precompiled && library.PrecompiledWithSources { + // Fully precompiled libraries should have no dependencies to avoid ABI breakage + if l.verbose { + l.verboseInfoFn(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) + } + } else { + for _, sourceDir := range library.SourceDirs() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) + } + } + first = false + } +} + +func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + sourceFileQueue *uniqueSourceFileQueue, + origin interface{}, + folder *paths.Path, + recurse bool, +) error { + sourceFileExtensions := []string{} + for k := range globals.SourceFilesValidExtensions { + sourceFileExtensions = append(sourceFileExtensions, k) + } + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) + if err != nil { + return errors.WithStack(err) + } + + for _, filePath := range filePaths { + sourceFile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.push(sourceFile) + } + + return nil +} + +func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { + if len(l.importedLibraries) == 0 { + return nil + } + + for _, library := range l.importedLibraries { + if !library.IsLegacy { + if library.InstallDir.Join("arch").IsDir() { + return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) + } + for _, propName := range libraries.MandatoryProperties { + if !library.Properties.ContainsKey(propName) { + return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) + } + } + if library.Layout == libraries.RecursiveLayout { + if library.UtilityDir != nil { + return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) + } + } + } + } + + return nil +} + +// includeRegexp fixdoc +var includeRegexp = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") + +// IncludesFinderWithRegExp fixdoc +func IncludesFinderWithRegExp(source string) string { + match := includeRegexp.FindStringSubmatch(source) + if match != nil { + return strings.TrimSpace(match[1]) + } + return findIncludeForOldCompilers(source) +} + +func findIncludeForOldCompilers(source string) string { + lines := strings.Split(source, "\n") + for _, line := range lines { + splittedLine := strings.Split(line, ":") + for i := range splittedLine { + if strings.Contains(splittedLine[i], "fatal error") { + return strings.TrimSpace(splittedLine[i+1]) + } + } + } + return "" +} + +type sourceFile struct { + // Path to the source file within the sketch/library root folder + relativePath *paths.Path + + // ExtraIncludePath contains an extra include path that must be + // used to compile this source file. + // This is mainly used for source files that comes from old-style libraries + // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. + extraIncludePath *paths.Path + + // The source root for the given origin, where its source files + // can be found. Prepending this to SourceFile.RelativePath will give + // the full path to that source file. + sourceRoot *paths.Path + + // The build root for the given origin, where build products will + // be placed. Any directories inside SourceFile.RelativePath will be + // appended here. + buildRoot *paths.Path +} + +// Equals fixdoc +func (f *sourceFile) Equals(g *sourceFile) bool { + return f.relativePath.EqualsTo(g.relativePath) && + f.buildRoot.EqualsTo(g.buildRoot) && + f.sourceRoot.EqualsTo(g.sourceRoot) +} + +// makeSourceFile containing the given source file path within the +// given origin. The given path can be absolute, or relative within the +// origin's root source folder +func makeSourceFile( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + origin interface{}, + path *paths.Path, +) (*sourceFile, error) { + res := &sourceFile{} + + switch o := origin.(type) { + case *sketch.Sketch: + res.buildRoot = sketchBuildPath + res.sourceRoot = sketchBuildPath + case *libraries.Library: + res.buildRoot = librariesBuildPath.Join(o.DirName) + res.sourceRoot = o.SourceDir + res.extraIncludePath = o.UtilityDir + default: + panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) + } + + if path.IsAbs() { + var err error + path, err = res.sourceRoot.RelTo(path) + if err != nil { + return nil, err + } + } + res.relativePath = path + return res, nil +} + +// ExtraIncludePath fixdoc +func (f *sourceFile) ExtraIncludePath() *paths.Path { + return f.extraIncludePath +} + +// SourcePath fixdoc +func (f *sourceFile) SourcePath() *paths.Path { + return f.sourceRoot.JoinPath(f.relativePath) +} + +// ObjectPath fixdoc +func (f *sourceFile) ObjectPath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".o") +} + +// DepfilePath fixdoc +func (f *sourceFile) DepfilePath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".d") +} + +// LibrariesLoader todo +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { + // Since we are using the cached libraries resolution + // the library manager is not needed. + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) + + builtInLibrariesFolders := builtInLibrariesDirs + if builtInLibrariesFolders != nil { + if err := builtInLibrariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) + } + + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) + } + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) + + librariesFolders := otherLibrariesDirs + if err := librariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + for _, folder := range librariesFolders { + lm.AddLibrariesDir(folder, libraries.User) + } + + for _, status := range lm.RescanLibraries() { + // With the refactoring of the initialization step of the CLI we changed how + // errors are returned when loading platforms and libraries, that meant returning a list of + // errors instead of a single one to enhance the experience for the user. + // I have no intention right now to start a refactoring of the legacy package too, so + // here's this shitty solution for now. + // When we're gonna refactor the legacy package this will be gone. + verboseOut.Write([]byte(status.Message())) + } + + for _, dir := range libraryDirs { + // Libraries specified this way have top priority + if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + } + + resolver := librariesresolver.NewCppResolver() + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + return lm, resolver, verboseOut.Bytes(), nil +} + +type includeCacheEntry struct { + Sourcefile *paths.Path + Include string + Includepath *paths.Path +} + +// String fixdoc +func (entry *includeCacheEntry) String() string { + return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", + entry.Sourcefile, entry.Include, entry.Includepath) +} + +// Equals fixdoc +func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { + return entry.String() == other.String() +} + +type includeCache struct { + // Are the cache contents valid so far? + valid bool + // Index into entries of the next entry to be processed. Unused + // when the cache is invalid. + next int + entries []*includeCacheEntry +} + +// Next Return the next cache entry. Should only be called when the cache is +// valid and a next entry is available (the latter can be checked with +// ExpectFile). Does not advance the cache. +func (cache *includeCache) Next() *includeCacheEntry { + return cache.entries[cache.next] +} + +// ExpectFile check that the next cache entry is about the given file. If it is +// not, or no entry is available, the cache is invalidated. Does not +// advance the cache. +func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { + if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// ExpectEntry check that the next entry matches the given values. If so, advance +// the cache. If not, the cache is invalidated. If the cache is +// invalidated, or was already invalid, an entry with the given values +// is appended. +func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { + entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} + if cache.valid { + if cache.next < len(cache.entries) && cache.Next().Equals(entry) { + cache.next++ + } else { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } + } + + if !cache.valid { + cache.entries = append(cache.entries, entry) + } +} + +// ExpectEnd check that the cache is completely consumed. If not, the cache is +// invalidated. +func (cache *includeCache) ExpectEnd() { + if cache.valid && cache.next < len(cache.entries) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Read the cache from the given file +func readCache(path *paths.Path) *includeCache { + bytes, err := path.ReadFile() + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result := &includeCache{} + err = json.Unmarshal(bytes, &result.entries) + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result.valid = true + return result +} + +// Write the given cache to the given file if it is invalidated. If the +// cache is still valid, just update the timestamps of the file. +func writeCache(cache *includeCache, path *paths.Path) error { + // If the cache was still valid all the way, just touch its file + // (in case any source file changed without influencing the + // includes). If it was invalidated, overwrite the cache with + // the new contents. + if cache.valid { + path.Chtimes(time.Now(), time.Now()) + } else { + bytes, err := json.MarshalIndent(cache.entries, "", " ") + if err != nil { + return errors.WithStack(err) + } + err = path.WriteFile(bytes) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +type uniqueSourceFileQueue []*sourceFile + +func (queue *uniqueSourceFileQueue) push(value *sourceFile) { + if !queue.contains(value) { + *queue = append(*queue, value) + } +} + +func (queue uniqueSourceFileQueue) contains(target *sourceFile) bool { + return slices.ContainsFunc(queue, target.Equals) +} + +func (queue *uniqueSourceFileQueue) pop() *sourceFile { + old := *queue + x := old[0] + *queue = old[1:] + return x +} + +func (queue uniqueSourceFileQueue) empty() bool { + return len(queue) == 0 +} diff --git a/arduino/builder/preprocessor/arduino_preprocessor.go b/arduino/builder/preprocessor/arduino_preprocessor.go new file mode 100644 index 00000000000..a81d4bff612 --- /dev/null +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -0,0 +1,92 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package preprocessor + +import ( + "bytes" + "context" + "path/filepath" + "runtime" + + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/executils" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +// PreprocessSketchWithArduinoPreprocessor performs preprocessing of the arduino sketch +// using arduino-preprocessor (https://github.com/arduino/arduino-preprocessor). +func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) { + verboseOut := &bytes.Buffer{} + normalOut := &bytes.Buffer{} + if err := buildPath.Join("preproc").MkdirAll(); err != nil { + return nil, nil, err + } + + sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp") + targetFile := buildPath.Join("preproc", "sketch_merged.cpp") + gccStdout, gccStderr, err := GCC(sourceFile, targetFile, includeFolders, buildProperties) + verboseOut.Write(gccStdout) + verboseOut.Write(gccStderr) + if err != nil { + return nil, nil, err + } + + arduiniPreprocessorProperties := properties.NewMap() + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) + arduiniPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") + arduiniPreprocessorProperties.Merge(buildProperties) + arduiniPreprocessorProperties.Merge(arduiniPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor")) + arduiniPreprocessorProperties.SetPath("source_file", targetFile) + pattern := arduiniPreprocessorProperties.Get("pattern") + if pattern == "" { + return nil, nil, errors.New(tr("arduino-preprocessor pattern is missing")) + } + + commandLine := arduiniPreprocessorProperties.ExpandPropsInString(pattern) + parts, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, nil, err + } + if runtime.GOOS == "windows" { + // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) + command.SetDir(filepath.VolumeName(parts[0]) + "/") + } + + verboseOut.WriteString(commandLine) + commandStdOut, commandStdErr, err := command.RunAndCaptureOutput(context.Background()) + verboseOut.Write(commandStdErr) + if err != nil { + return normalOut.Bytes(), verboseOut.Bytes(), err + } + result := utils.NormalizeUTF8(commandStdOut) + + destFile := buildPath.Join(sk.MainFile.Base() + ".cpp") + if err := destFile.WriteFile(result); err != nil { + return normalOut.Bytes(), verboseOut.Bytes(), err + } + + return normalOut.Bytes(), verboseOut.Bytes(), err +} diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 476c1f2f5b4..2029f08b972 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -38,7 +38,7 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths. gccBuildProperties.SetPath("source_file", sourceFilePath) gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath) - includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) + includesStrings := f.Map(includes.AsStrings(), cpp.WrapWithHyphenI) gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) const gccPreprocRecipeProperty = "recipe.preproc.macros" diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index a79bbac4bef..144164cf31b 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -21,7 +21,6 @@ import ( "regexp" "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -37,42 +36,36 @@ var ( // PrepareSketchBuildPath copies the sketch source files in the build path. // The .ino files are merged together to create a .cpp file (by the way, the // .cpp file still needs to be Arduino-preprocessed to compile). -func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { - if offset, mergedSource, err := sketchMergeSources(sketch, sourceOverrides); err != nil { - return 0, err - } else if err := SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { - return 0, err - } else if err := sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { - return 0, err - } else { - return offset, nil +func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { + if err := buildPath.MkdirAll(); err != nil { + return 0, errors.Wrap(err, tr("unable to create a folder to save the sketch")) } -} -// SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk -func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path) error { - sketchName := path.Base() - if err := destPath.MkdirAll(); err != nil { - return errors.Wrap(err, tr("unable to create a folder to save the sketch")) + offset, mergedSource, err := b.sketchMergeSources(sourceOverrides) + if err != nil { + return 0, err } - destFile := destPath.Join(fmt.Sprintf("%s.cpp", sketchName)) + destFile := buildPath.Join(b.sketch.MainFile.Base() + ".cpp") + if err := destFile.WriteFile([]byte(mergedSource)); err != nil { + return 0, err + } - if err := destFile.WriteFile(contents); err != nil { - return errors.Wrap(err, tr("unable to save the sketch on disk")) + if err := b.sketchCopyAdditionalFiles(buildPath, sourceOverrides); err != nil { + return 0, err } - return nil + return offset, nil } // sketchMergeSources merges all the .ino source files included in a sketch to produce // a single .cpp file. -func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { +func (b *Builder) sketchMergeSources(overrides map[string]string) (int, string, error) { lineOffset := 0 mergedSource := "" getSource := func(f *paths.Path) (string, error) { - path, err := sk.FullPath.RelTo(f) + path, err := b.sketch.FullPath.RelTo(f) if err != nil { return "", errors.Wrap(err, tr("unable to compute relative path to the sketch for the item")) } @@ -87,7 +80,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st } // add Arduino.h inclusion directive if missing - mainSrc, err := getSource(sk.MainFile) + mainSrc, err := getSource(b.sketch.MainFile) if err != nil { return 0, "", err } @@ -96,11 +89,11 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st lineOffset++ } - mergedSource += "#line 1 " + cpp.QuoteString(sk.MainFile.String()) + "\n" + mergedSource += "#line 1 " + cpp.QuoteString(b.sketch.MainFile.String()) + "\n" mergedSource += mainSrc + "\n" lineOffset++ - for _, file := range sk.OtherSketchFiles { + for _, file := range b.sketch.OtherSketchFiles { src, err := getSource(file) if err != nil { return 0, "", err @@ -114,18 +107,14 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st // sketchCopyAdditionalFiles copies the additional files for a sketch to the // specified destination directory. -func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { - if err := destPath.MkdirAll(); err != nil { - return errors.Wrap(err, tr("unable to create a folder to save the sketch files")) - } - - for _, file := range sketch.AdditionalFiles { - relpath, err := sketch.FullPath.RelTo(file) +func (b *Builder) sketchCopyAdditionalFiles(buildPath *paths.Path, overrides map[string]string) error { + for _, file := range b.sketch.AdditionalFiles { + relpath, err := b.sketch.FullPath.RelTo(file) if err != nil { return errors.Wrap(err, tr("unable to compute relative path to the sketch for the item")) } - targetPath := destPath.JoinPath(relpath) + targetPath := buildPath.JoinPath(relpath) // create the directory containing the target if err = targetPath.Parent().MkdirAll(); err != nil { return errors.Wrap(err, tr("unable to create the folder containing the item")) @@ -180,16 +169,16 @@ func writeIfDifferent(source []byte, destPath *paths.Path) error { // SetupBuildProperties adds the build properties related to the sketch to the // default board build properties map. -func SetupBuildProperties(boardBuildProperties *properties.Map, buildPath *paths.Path, sketch *sketch.Sketch, optimizeForDebug bool) *properties.Map { +func (b *Builder) SetupBuildProperties(boardBuildProperties *properties.Map, buildPath *paths.Path, optimizeForDebug bool) *properties.Map { buildProperties := properties.NewMap() buildProperties.Merge(boardBuildProperties) if buildPath != nil { buildProperties.SetPath("build.path", buildPath) } - if sketch != nil { - buildProperties.Set("build.project_name", sketch.MainFile.Base()) - buildProperties.SetPath("build.source.path", sketch.FullPath) + if b.sketch != nil { + buildProperties.Set("build.project_name", b.sketch.MainFile.Base()) + buildProperties.SetPath("build.source.path", b.sketch.FullPath) } if optimizeForDebug { if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok { diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index 5466cdff819..b1fda4e6e8c 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -17,8 +17,6 @@ package builder import ( "fmt" - "os" - "path/filepath" "runtime" "strings" "testing" @@ -28,40 +26,11 @@ import ( "github.com/stretchr/testify/require" ) -func tmpDirOrDie() *paths.Path { - dir, err := os.MkdirTemp(os.TempDir(), "builder_test") - if err != nil { - panic(fmt.Sprintf("error creating tmp dir: %v", err)) - } - return paths.New(dir) -} - -func TestSaveSketch(t *testing.T) { - sketchName := t.Name() + ".ino" - outName := sketchName + ".cpp" - sketchFile := filepath.Join("testdata", sketchName) - tmp := tmpDirOrDie() - defer tmp.RemoveAll() - source, err := os.ReadFile(sketchFile) - if err != nil { - t.Fatalf("unable to read golden file %s: %v", sketchFile, err) - } - - SketchSaveItemCpp(paths.New(sketchName), source, tmp) - - out, err := tmp.Join(outName).ReadFile() - if err != nil { - t.Fatalf("unable to read output file %s: %v", outName, err) - } - - require.Equal(t, source, out) -} - func TestMergeSketchSources(t *testing.T) { // borrow the sketch from TestLoadSketchFolder to avoid boilerplate - s, err := sketch.New(paths.New("testdata", "TestLoadSketchFolder")) + sk, err := sketch.New(paths.New("testdata", "TestLoadSketchFolder")) require.Nil(t, err) - require.NotNil(t, s) + require.NotNil(t, sk) // load expected result suffix := ".txt" @@ -69,66 +38,68 @@ func TestMergeSketchSources(t *testing.T) { suffix = "_win.txt" } mergedPath := paths.New("testdata", t.Name()+suffix) + require.NoError(t, mergedPath.ToAbs()) mergedBytes, err := mergedPath.ReadFile() - if err != nil { - t.Fatalf("unable to read golden file %s: %v", mergedPath, err) - } + require.NoError(t, err, "reading golden file %s: %v", mergedPath, err) - mergedPath.ToAbs() pathToGoldenSource := mergedPath.Parent().Parent().String() if runtime.GOOS == "windows" { pathToGoldenSource = strings.ReplaceAll(pathToGoldenSource, `\`, `\\`) } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - offset, source, err := sketchMergeSources(s, nil) + b := NewBuilder(sk) + offset, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 2, offset) require.Equal(t, mergedSources, source) } func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { - s, err := sketch.New(paths.New("testdata", t.Name())) + sk, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) - require.NotNil(t, s) + require.NotNil(t, sk) // ensure not to include Arduino.h when it's already there - _, source, err := sketchMergeSources(s, nil) + b := NewBuilder(sk) + _, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) } func TestCopyAdditionalFiles(t *testing.T) { - tmp := tmpDirOrDie() + tmp, err := paths.MkTempDir("", "") + require.NoError(t, err) defer tmp.RemoveAll() // load the golden sketch - s1, err := sketch.New(paths.New("testdata", t.Name())) + sk1, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) - require.Equal(t, s1.AdditionalFiles.Len(), 1) + require.Equal(t, sk1.AdditionalFiles.Len(), 1) + b1 := NewBuilder(sk1) // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later - err = sketchCopyAdditionalFiles(s1, tmp, nil) + err = b1.sketchCopyAdditionalFiles(tmp, nil) require.Nil(t, err) fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base())) require.Nil(t, fakeIno.WriteFile([]byte{})) // compare - s2, err := sketch.New(tmp) + sk2, err := sketch.New(tmp) require.Nil(t, err) - require.Equal(t, s2.AdditionalFiles.Len(), 1) + require.Equal(t, sk2.AdditionalFiles.Len(), 1) // save file info - info1, err := s2.AdditionalFiles[0].Stat() + info1, err := sk2.AdditionalFiles[0].Stat() require.Nil(t, err) // copy again - err = sketchCopyAdditionalFiles(s1, tmp, nil) + err = b1.sketchCopyAdditionalFiles(tmp, nil) require.Nil(t, err) // verify file hasn't changed - info2, err := s2.AdditionalFiles[0].Stat() + info2, err := sk2.AdditionalFiles[0].Stat() require.NoError(t, err) require.Equal(t, info1.ModTime(), info2.ModTime()) } diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go new file mode 100644 index 00000000000..ae64dacb47c --- /dev/null +++ b/arduino/builder/utils/utils.go @@ -0,0 +1,181 @@ +package utils + +import ( + "os" + "strings" + "unicode" + + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +// ObjFileIsUpToDate fixdoc +func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { + logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) + if objectFile == nil || dependencyFile == nil { + logrus.Debugf("Not found: nil") + return false, nil + } + + sourceFile = sourceFile.Clean() + sourceFileStat, err := sourceFile.Stat() + if err != nil { + return false, errors.WithStack(err) + } + + objectFile = objectFile.Clean() + objectFileStat, err := objectFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", objectFile) + return false, nil + } + return false, errors.WithStack(err) + } + + dependencyFile = dependencyFile.Clean() + dependencyFileStat, err := dependencyFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", dependencyFile) + return false, nil + } + return false, errors.WithStack(err) + } + + if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, objectFile) + return false, nil + } + if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) + return false, nil + } + + rows, err := dependencyFile.ReadFileAsLines() + if err != nil { + return false, errors.WithStack(err) + } + + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) + + if len(rows) == 0 { + return true, nil + } + + firstRow := rows[0] + if !strings.HasSuffix(firstRow, ":") { + logrus.Debugf("No colon in first line of depfile") + return false, nil + } + objFileInDepFile := firstRow[:len(firstRow)-1] + if objFileInDepFile != objectFile.String() { + logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) + return false, nil + } + + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + return false, nil + } + + rows = rows[1:] + for _, row := range rows { + depStat, err := os.Stat(row) + if err != nil && !os.IsNotExist(err) { + // There is probably a parsing error of the dep file + // Ignore the error and trigger a full rebuild anyway + logrus.WithError(err).Debugf("Failed to read: %v", row) + return false, nil + } + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", row) + return false, nil + } + if depStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", row, objectFile) + return false, nil + } + } + + return true, nil +} + +func removeEndingBackSlash(s string) string { + return strings.TrimSuffix(s, "\\") +} + +func unescapeDep(s string) string { + s = strings.ReplaceAll(s, "\\ ", " ") + s = strings.ReplaceAll(s, "\\\t", "\t") + s = strings.ReplaceAll(s, "\\#", "#") + s = strings.ReplaceAll(s, "$$", "$") + s = strings.ReplaceAll(s, "\\\\", "\\") + return s +} + +// NormalizeUTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + +var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} + +// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func FilterOutSCCS(file *paths.Path) bool { + return !sourceControlFolders[file.Base()] +} + +// FilterReadableFiles is a ReadDirFilter that accepts only readable files +func FilterReadableFiles(file *paths.Path) bool { + // See if the file is readable by opening it + f, err := file.Open() + if err != nil { + return false + } + f.Close() + return true +} + +// filterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var filterOutHiddenFiles = paths.FilterOutPrefixes(".") + +// FindFilesInFolder fixdoc +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { + fileFilter := paths.AndFilter( + filterOutHiddenFiles, + FilterOutSCCS, + paths.FilterOutDirectories(), + FilterReadableFiles, + ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } + if recurse { + dirFilter := paths.AndFilter( + filterOutHiddenFiles, + FilterOutSCCS, + ) + return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) + } + return dir.ReadDir(fileFilter) +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 92858dd68d6..1daa6f31714 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -24,7 +24,9 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/buildcache" @@ -33,6 +35,7 @@ import ( "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/internal/inventory" "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -152,8 +155,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream // cache is purged after compilation to not remove entries that might be required defer maybePurgeBuildCache() + sketchBuilder := bldr.NewBuilder(sk) + // Add build properites related to sketch data - buildProperties = bldr.SetupBuildProperties(buildProperties, buildPath, sk, req.GetOptimizeForDebug()) + buildProperties = sketchBuilder.SetupBuildProperties(buildProperties, buildPath, req.GetOptimizeForDebug()) // Add user provided custom build properties customBuildPropertiesArgs := append(req.GetBuildProperties(), "build.warn_data_percentage=75") @@ -169,10 +174,8 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } builderCtx := &types.Context{} + builderCtx.Builder = sketchBuilder builderCtx.PackageManager = pme - if pme.GetProfile() != nil { - builderCtx.LibrariesManager = lm - } builderCtx.TargetBoard = targetBoard builderCtx.TargetPlatform = targetPlatform builderCtx.TargetPackage = targetPackage @@ -180,7 +183,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.RequiredTools = requiredTools builderCtx.BuildProperties = buildProperties builderCtx.CustomBuildProperties = customBuildPropertiesArgs - builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() builderCtx.FQBN = fqbn builderCtx.Sketch = sk builderCtx.BuildPath = buildPath @@ -199,7 +201,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Verbose = req.GetVerbose() builderCtx.Jobs = int(req.GetJobs()) + builderCtx.WarningsLevel = req.GetWarnings() + if builderCtx.WarningsLevel == "" { + builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL + } if req.GetBuildCachePath() == "" { builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") @@ -222,6 +228,57 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly() builderCtx.SourceOverride = req.GetSourceOverride() + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + builderCtx.SketchBuildPath = sketchBuildPath + builderCtx.LibrariesBuildPath = librariesBuildPath + builderCtx.CoreBuildPath = coreBuildPath + + if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { + return r, &arduino.CompileFailedError{ + Message: tr("Sketch cannot be located in build path. Please specify a different build path"), + } + } + + var libsManager *librariesmanager.LibrariesManager + if pme.GetProfile() != nil { + libsManager = lm + } + useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( + useCachedLibrariesResolution, libsManager, + builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, + builderCtx.ActualPlatform, builderCtx.TargetPlatform, + ) + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + + if builderCtx.Verbose { + builderCtx.Warn(string(verboseOut)) + } + + builderCtx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( + libsManager, libsResolver, + builderCtx.Verbose, + useCachedLibrariesResolution, + req.GetCreateCompilationDatabaseOnly(), + func(msg string) { builderCtx.Info(msg) }, + func(msg string) { builderCtx.Warn(msg) }, + func(data []byte) { builderCtx.WriteStdout(data) }, + func(data []byte) { builderCtx.WriteStderr(data) }, + ) + defer func() { if p := builderCtx.BuildPath; p != nil { r.BuildPath = p.String() @@ -243,13 +300,9 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } }() + // Just get build properties and exit if req.GetShowProperties() { - // Just get build properties and exit - compileErr := builder.RunParseHardware(builderCtx) - if compileErr != nil { - compileErr = &arduino.CompileFailedError{Message: compileErr.Error()} - } - return r, compileErr + return r, nil } if req.GetPreprocess() { @@ -263,7 +316,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream defer func() { importedLibs := []*rpc.Library{} - for _, lib := range builderCtx.ImportedLibraries { + for _, lib := range builderCtx.SketchLibrariesDetector.ImportedLibraries() { rpcLib, err := lib.ToRPCLibrary() if err != nil { msg := tr("Error getting information for library %s", lib.Name) + ": " + err.Error() + "\n" diff --git a/executils/process.go b/executils/process.go index 7e0ac135089..137d2f86bc3 100644 --- a/executils/process.go +++ b/executils/process.go @@ -137,6 +137,11 @@ func (p *Process) SetDir(dir string) { p.cmd.Dir = dir } +// GetDir gets the working directory of the command. +func (p *Process) GetDir() string { + return p.cmd.Dir +} + // SetDirFromPath sets the working directory of the command. If path is nil, Run // runs the command in the calling process's current directory. func (p *Process) SetDirFromPath(path *paths.Path) { @@ -187,3 +192,8 @@ func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, erro err := p.RunWithinContext(ctx) return stdout.Bytes(), stderr.Bytes(), err } + +// GetArgs returns the command arguments +func (p *Process) GetArgs() []string { + return p.cmd.Args +} diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go deleted file mode 100644 index 9e9fdd80669..00000000000 --- a/legacy/builder/add_additional_entries_to_context.go +++ /dev/null @@ -1,54 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type AddAdditionalEntriesToContext struct{} - -func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { - if ctx.BuildPath != nil { - buildPath := ctx.BuildPath - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return errors.WithStack(err) - } - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return errors.WithStack(err) - } - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return errors.WithStack(err) - } - - ctx.SketchBuildPath = sketchBuildPath - ctx.LibrariesBuildPath = librariesBuildPath - ctx.CoreBuildPath = coreBuildPath - } - - if ctx.WarningsLevel == "" { - ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL - } - - ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} - - return nil -} diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 90d92938811..ff517a5a2d4 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -19,7 +19,6 @@ import ( "reflect" "time" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" @@ -43,19 +42,17 @@ func (s *Builder) Run(ctx *types.Context) error { var _err, mainErr error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -92,7 +89,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, } - ctx.Progress.AddSubSteps(len(commands) + 4) + ctx.Progress.AddSubSteps(len(commands) + 5) defer ctx.Progress.RemoveSubSteps() for _, command := range commands { @@ -112,7 +109,10 @@ func (s *Builder) Run(ctx *types.Context) error { var otherErr error commands = []types.Command{ - &PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil}, + types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.PrintUsedAndNotUsedLibraries(mainErr != nil) + return nil + }), &PrintUsedLibrariesIfVerbose{}, @@ -139,19 +139,19 @@ func (s *Builder) Run(ctx *types.Context) error { } func PreprocessSketch(ctx *types.Context) error { + preprocessorImpl := preprocessor.PreprocessSketchWithCtags if ctx.UseArduinoPreprocessor { - return PreprocessSketchWithArduinoPreprocessor(ctx) + preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor + } + normalOutput, verboseOutput, err := preprocessorImpl( + ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, + ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) + if ctx.Verbose { + ctx.WriteStdout(verboseOutput) } else { - normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, - ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) - if ctx.Verbose { - ctx.WriteStdout(verboseOutput) - } else { - ctx.WriteStdout(normalOutput) - } - return err + ctx.WriteStdout(normalOutput) } + return err } type Preprocess struct{} @@ -163,18 +163,16 @@ func (s *Preprocess) Run(ctx *types.Context) error { var _err error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -218,14 +216,22 @@ func RunBuilder(ctx *types.Context) error { return runCommands(ctx, []types.Command{&Builder{}}) } -func RunParseHardware(ctx *types.Context) error { - commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - return runCommands(ctx, commands) -} - func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) } + +func findIncludes(ctx *types.Context) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 4a227e76154..1d53860b163 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -18,22 +18,21 @@ package builder_utils import ( "fmt" "os" - "os/exec" "path/filepath" "runtime" "strings" "sync" + bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" - f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) var tr = i18n.Tr @@ -48,7 +47,7 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } targetModTime := targetStat.ModTime() - files, err := utils.FindFilesInFolder(dir, true, extensions...) + files, err := bUtils.FindFilesInFolder(dir, true, extensions...) if err != nil { return false, err } @@ -64,32 +63,25 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st return true, nil } -func CompileFiles(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourcePath, false, buildPath, buildProperties, includes) +func CompileFiles(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { + return compileFiles(ctx, sourceDir, false, buildPath, buildProperties, includes) } -func CompileFilesRecursive(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourcePath, true, buildPath, buildProperties, includes) +func CompileFilesRecursive(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { + return compileFiles(ctx, sourceDir, true, buildPath, buildProperties, includes) } -func compileFiles(ctx *types.Context, sourcePath *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - var sources paths.PathList - var err error - if recurse { - sources, err = sourcePath.ReadDirRecursive() - } else { - sources, err = sourcePath.ReadDir() - } - if err != nil { - return nil, err - } - +func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { validExtensions := []string{} for ext := range globals.SourceFilesValidExtensions { validExtensions = append(validExtensions, ext) } - sources.FilterSuffix(validExtensions...) + sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + ctx.Progress.AddSubSteps(len(sources)) defer ctx.Progress.RemoveSubSteps() @@ -107,7 +99,7 @@ func compileFiles(ctx *types.Context, sourcePath *paths.Path, recurse bool, buil if !buildProperties.ContainsKey(recipe) { recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) } - objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) + objectFile, err := compileFileWithRecipe(ctx, sourceDir, source, buildPath, buildProperties, includes, recipe) if err != nil { errorsMux.Lock() errorsList = append(errorsList, err) @@ -176,12 +168,12 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { return nil, errors.WithStack(err) } - command, err := PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, recipe, false) if err != nil { return nil, errors.WithStack(err) } @@ -212,120 +204,6 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return objectFile, nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { - logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) - if objectFile == nil || dependencyFile == nil { - logrus.Debugf("Not found: nil") - return false, nil - } - - sourceFile = sourceFile.Clean() - sourceFileStat, err := sourceFile.Stat() - if err != nil { - return false, errors.WithStack(err) - } - - objectFile = objectFile.Clean() - objectFileStat, err := objectFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", objectFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - dependencyFile = dependencyFile.Clean() - dependencyFileStat, err := dependencyFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", dependencyFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, objectFile) - return false, nil - } - if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) - return false, nil - } - - rows, err := dependencyFile.ReadFileAsLines() - if err != nil { - return false, errors.WithStack(err) - } - - rows = f.Map(rows, removeEndingBackSlash) - rows = f.Map(rows, strings.TrimSpace) - rows = f.Map(rows, unescapeDep) - rows = f.Filter(rows, f.NotEquals("")) - - if len(rows) == 0 { - return true, nil - } - - firstRow := rows[0] - if !strings.HasSuffix(firstRow, ":") { - logrus.Debugf("No colon in first line of depfile") - return false, nil - } - objFileInDepFile := firstRow[:len(firstRow)-1] - if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) - return false, nil - } - - // The first line of the depfile contains the path to the object file to generate. - // The second line of the depfile contains the path to the source file. - // All subsequent lines contain the header files necessary to compile the object file. - - // If we don't do this check it might happen that trying to compile a source file - // that has the same name but a different path wouldn't recreate the object file. - if sourceFile.String() != strings.Trim(rows[1], " ") { - return false, nil - } - - rows = rows[1:] - for _, row := range rows { - depStat, err := os.Stat(row) - if err != nil && !os.IsNotExist(err) { - // There is probably a parsing error of the dep file - // Ignore the error and trigger a full rebuild anyway - logrus.WithError(err).Debugf("Failed to read: %v", row) - return false, nil - } - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", row) - return false, nil - } - if depStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", row, objectFile) - return false, nil - } - } - - return true, nil -} - -func unescapeDep(s string) string { - s = strings.Replace(s, "\\ ", " ", -1) - s = strings.Replace(s, "\\\t", "\t", -1) - s = strings.Replace(s, "\\#", "#", -1) - s = strings.Replace(s, "$$", "$", -1) - s = strings.Replace(s, "\\\\", "\\", -1) - return s -} - -func removeEndingBackSlash(s string) string { - return strings.TrimSuffix(s, "\\") -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) @@ -366,7 +244,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath) properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) - command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return nil, errors.WithStack(err) } @@ -382,7 +260,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile const COMMANDLINE_LIMIT = 30000 -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool, toolEnv []string) (*exec.Cmd, error) { +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { pattern := buildProperties.Get(recipe) if pattern == "" { return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) @@ -397,24 +275,30 @@ func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, rem if err != nil { return nil, errors.WithStack(err) } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), toolEnv...) // if the overall commandline is too long for the platform // try reducing the length by making the filenames relative // and changing working directory to build.path + var relativePath string if len(commandLine) > COMMANDLINE_LIMIT { - relativePath := buildProperties.Get("build.path") - for i, arg := range command.Args { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { if _, err := os.Stat(arg); os.IsNotExist(err) { continue } rel, err := filepath.Rel(relativePath, arg) if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - command.Args[i] = rel + parts[i] = rel } } - command.Dir = relativePath + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) } return command, nil diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go deleted file mode 100644 index 5d849b927ed..00000000000 --- a/legacy/builder/container_find_includes.go +++ /dev/null @@ -1,515 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -/* - -Include detection - -This code is responsible for figuring out what libraries the current -sketch needs an populates both Context.ImportedLibraries with a list of -Library objects, and Context.IncludeFolders with a list of folders to -put on the include path. - -Simply put, every #include in a source file pulls in the library that -provides that source file. This includes source files in the selected -libraries, so libraries can recursively include other libraries as well. - -To implement this, the gcc preprocessor is used. A queue is created -containing, at first, the source files in the sketch. Each of the files -in the queue is processed in turn by running the preprocessor on it. If -the preprocessor provides an error, the output is examined to see if the -error is a missing header file originating from a #include directive. - -The filename is extracted from that #include directive, and a library is -found that provides it. If multiple libraries provide the same file, one -is slected (how this selection works is not described here, see the -ResolveLibrary function for that). The library selected in this way is -added to the include path through Context.IncludeFolders and the list of -libraries to include in the link through Context.ImportedLibraries. - -Furthermore, all of the library source files are added to the queue, to -be processed as well. When the preprocessor completes without showing an -#include error, processing of the file is complete and it advances to -the next. When no library can be found for a included filename, an error -is shown and the process is aborted. - -Caching - -Since this process is fairly slow (requiring at least one invocation of -the preprocessor per source file), its results are cached. - -Just caching the complete result (i.e. the resulting list of imported -libraries) seems obvious, but such a cache is hard to invalidate. Making -a list of all the source and header files used to create the list and -check if any of them changed is probably feasible, but this would also -require caching the full list of libraries to invalidate the cache when -the include to library resolution might have a different result. Another -downside of a complete cache is that any changes requires re-running -everything, even if no includes were actually changed. - -Instead, caching happens by keeping a sort of "journal" of the steps in -the include detection, essentially tracing each file processed and each -include path entry added. The cache is used by retracing these steps: -The include detection process is executed normally, except that instead -of running the preprocessor, the include filenames are (when possible) -read from the cache. Then, the include file to library resolution is -again executed normally. The results are checked against the cache and -as long as the results match, the cache is considered valid. - -When a source file (or any of the files it includes, as indicated by the -.d file) is changed, the preprocessor is executed as normal for the -file, ignoring any includes from the cache. This does not, however, -invalidate the cache: If the results from the preprocessor match the -entries in the cache, the cache remains valid and can again be used for -the next (unchanged) file. - -The cache file uses the JSON format and contains a list of entries. Each -entry represents a discovered library and contains: - - Sourcefile: The source file that the include was found in - - Include: The included filename found - - Includepath: The addition to the include path - -There are also some special entries: - - When adding the initial include path entries, such as for the core - and variant paths. These are not discovered, so the Sourcefile and - Include fields will be empty. - - When a file contains no (more) missing includes, an entry with an - empty Include and IncludePath is generated. - -*/ - -package builder - -import ( - "encoding/json" - "fmt" - "os/exec" - "time" - - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ContainerFindIncludes struct{} - -func (s *ContainerFindIncludes) Run(ctx *types.Context) error { - err := s.findIncludes(ctx) - if err != nil && ctx.OnlyUpdateCompilationDatabase { - ctx.Info( - fmt.Sprintf("%s: %s", - tr("An error occurred detecting libraries"), - tr("the compilation database may be incomplete or inaccurate"))) - return nil - } - return err -} - -func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { - librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.UseCachedLibrariesResolution && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { - return err - } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { - return err - } - if ctx.Verbose { - ctx.Info("Using cached library discovery: " + librariesResolutionCache.String()) - } - return nil - } - - cachePath := ctx.BuildPath.Join("includes.cache") - cache := readCache(cachePath) - - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.core.path")) - if ctx.BuildProperties.Get("build.variant.path") != "" { - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) - } - - sourceFileQueue := &types.UniqueSourceFileQueue{} - - if !ctx.UseCachedLibrariesResolution { - sketch := ctx.Sketch - mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(mergedfile) - - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */) - srcSubfolderPath := ctx.SketchBuildPath.Join("src") - if srcSubfolderPath.IsDir() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) - } - - for !sourceFileQueue.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFileQueue) - if err != nil { - cachePath.Remove() - return errors.WithStack(err) - } - } - - // Finalize the cache - cache.ExpectEnd() - if err := writeCache(cache, cachePath); err != nil { - return errors.WithStack(err) - } - } - - if err := failIfImportedLibraryIsWrong(ctx); err != nil { - return errors.WithStack(err) - } - - if d, err := json.Marshal(ctx.IncludeFolders); err != nil { - return err - } else if err := librariesResolutionCache.WriteFile(d); err != nil { - return err - } - - return nil -} - -// Append the given folder to the include path and match or append it to -// the cache. sourceFilePath and include indicate the source of this -// include (e.g. what #include line in what file it was resolved from) -// and should be the empty string for the default include folders, like -// the core or variant. -func appendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { - ctx.IncludeFolders = append(ctx.IncludeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) -} - -type includeCacheEntry struct { - Sourcefile *paths.Path - Include string - Includepath *paths.Path -} - -func (entry *includeCacheEntry) String() string { - return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", - entry.Sourcefile, entry.Include, entry.Includepath) -} - -func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { - return entry.String() == other.String() -} - -type includeCache struct { - // Are the cache contents valid so far? - valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// Check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// Check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Read the cache from the given file -func readCache(path *paths.Path) *includeCache { - bytes, err := path.ReadFile() - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result.valid = true - return result -} - -// Write the given cache to the given file if it is invalidated. If the -// cache is still valid, just update the timestamps of the file. -func writeCache(cache *includeCache, path *paths.Path) error { - // If the cache was still valid all the way, just touch its file - // (in case any source file changed without influencing the - // includes). If it was invalidated, overwrite the cache with - // the new contents. - if cache.valid { - path.Chtimes(time.Now(), time.Now()) - } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") - if err != nil { - return errors.WithStack(err) - } - err = path.WriteFile(bytes) - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - -func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error { - sourceFile := sourceFileQueue.Pop() - sourcePath := sourceFile.SourcePath() - targetFilePath := paths.NullPath() - depPath := sourceFile.DepfilePath() - objPath := sourceFile.ObjectPath() - - // TODO: This should perhaps also compare against the - // include.cache file timestamp. Now, it only checks if the file - // changed after the object file was generated, but if it - // changed between generating the cache and the object file, - // this could show the file as unchanged when it really is - // changed. Changing files during a build isn't really - // supported, but any problems from it should at least be - // resolved when doing another build, which is not currently the - // case. - // TODO: This reads the dependency file, but the actual building - // does it again. Should the result be somehow cached? Perhaps - // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) - if err != nil { - return errors.WithStack(err) - } - - first := true - for { - cache.ExpectFile(sourcePath) - - // Libraries may require the "utility" directory to be added to the include - // search path, but only for the source code of the library, so we temporary - // copy the current search path list and add the library' utility directory - // if needed. - includeFolders := ctx.IncludeFolders - if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { - includeFolders = append(includeFolders, extraInclude) - } - - var preprocErr error - var preprocStderr []byte - - var missingIncludeH string - if unchanged && cache.valid { - missingIncludeH = cache.Next().Include - if first && ctx.Verbose { - ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) - } - } else { - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - // Unwrap error and see if it is an ExitError. - if preprocErr == nil { - // Preprocessor successful, done - missingIncludeH = "" - } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { - // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors - return errors.WithStack(preprocErr) - } else { - missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) - if missingIncludeH == "" && ctx.Verbose { - ctx.Info(tr("Error while detecting libraries included by %[1]s", sourcePath)) - } - } - } - - if missingIncludeH == "" { - // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) - return nil - } - - library := ResolveLibrary(ctx, missingIncludeH) - if library == nil { - // Library could not be resolved, show error - if preprocErr == nil || preprocStderr == nil { - // Filename came from cache, so run preprocessor to obtain error to show - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - if preprocErr == nil { - // If there is a missing #include in the cache, but running - // gcc does not reproduce that, there is something wrong. - // Returning an error here will cause the cache to be - // deleted, so hopefully the next compilation will succeed. - return errors.New(tr("Internal error in cache")) - } - } - ctx.WriteStderr(preprocStderr) - return errors.WithStack(preprocErr) - } - - // Add this library to the list of libraries, the - // include path and queue its source files for further - // include scanning - ctx.ImportedLibraries = append(ctx.ImportedLibraries, library) - appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) - - if library.Precompiled && library.PrecompiledWithSources { - // Fully precompiled libraries should have no dependencies to avoid ABI breakage - if ctx.Verbose { - ctx.Info(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) - } - } else { - for _, sourceDir := range library.SourceDirs() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) - } - } - first = false - } -} - -func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { - sourceFileExtensions := []string{} - for k := range globals.SourceFilesValidExtensions { - sourceFileExtensions = append(sourceFileExtensions, k) - } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) - if err != nil { - return errors.WithStack(err) - } - - for _, filePath := range filePaths { - sourceFile, err := types.MakeSourceFile(ctx, origin, filePath) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(sourceFile) - } - - return nil -} - -func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - resolver := ctx.LibrariesResolver - importedLibraries := ctx.ImportedLibraries - - candidates := resolver.AlternativesFor(header) - - if ctx.Verbose { - ctx.Info(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - ctx.Info(fmt.Sprintf("ResolveLibrary(%s)", header)) - ctx.Info(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := resolver.ResolveFor(header, ctx.TargetPlatform.Platform.Architecture) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - ctx.LibrariesResolutionResults[header] = types.LibraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected -} - -func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.ImportedLibraries) == 0 { - return nil - } - - for _, library := range ctx.ImportedLibraries { - if !library.IsLegacy { - if library.InstallDir.Join("arch").IsDir() { - return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) - } - for _, propName := range libraries.MandatoryProperties { - if !library.Properties.ContainsKey(propName) { - return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) - } - } - if library.Layout == libraries.RecursiveLayout { - if library.UtilityDir != nil { - return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) - } - } - } - } - - return nil -} diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go deleted file mode 100644 index 3a00ff6edaa..00000000000 --- a/legacy/builder/container_setup.go +++ /dev/null @@ -1,43 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type ContainerSetupHardwareToolsLibsSketchAndProps struct{} - -func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - commands := []types.Command{ - &AddAdditionalEntriesToContext{}, - &FailIfBuildPathEqualsSketchPath{}, - &LibrariesLoader{}, - } - ctx.Progress.AddSubSteps(len(commands)) - defer ctx.Progress.RemoveSubSteps() - for _, command := range commands { - PrintRingNameIfDebug(ctx, command) - err := command.Run(ctx) - if err != nil { - return errors.WithStack(err) - } - ctx.Progress.CompleteStep() - ctx.PushProgress() - } - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index acbce09e2cb..6d6b67ae1d6 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -26,11 +26,11 @@ import ( properties "github.com/arduino/go-properties-orderedmap" "golang.org/x/exp/slices" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" ) type ExportProjectCMake struct { @@ -198,7 +198,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { cmakeFile := cmakeFolder.Join("CMakeLists.txt") dynamicLibsFromPkgConfig := map[string]bool{} - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { // Copy used libraries in the correct folder libDir := libBaseFolder.Join(library.DirName) mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) @@ -367,9 +367,9 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) - for _, arg := range command.Args { + for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { *defines = appendIfNotPresent(*defines, arg) continue diff --git a/legacy/builder/includes_finder_with_regexp.go b/legacy/builder/includes_finder_with_regexp.go deleted file mode 100644 index 03017c67048..00000000000 --- a/legacy/builder/includes_finder_with_regexp.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "regexp" - "strings" -) - -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") - -func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) - if match != nil { - return strings.TrimSpace(match[1]) - } - return findIncludeForOldCompilers(source) -} - -func findIncludeForOldCompilers(source string) string { - lines := strings.Split(source, "\n") - for _, line := range lines { - splittedLine := strings.Split(line, ":") - for i := range splittedLine { - if strings.Contains(splittedLine[i], "fatal error") { - return strings.TrimSpace(splittedLine[i+1]) - } - } - } - return "" -} diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go deleted file mode 100644 index b41c49edcb1..00000000000 --- a/legacy/builder/libraries_loader.go +++ /dev/null @@ -1,97 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type LibrariesLoader struct{} - -func (s *LibrariesLoader) Run(ctx *types.Context) error { - if ctx.UseCachedLibrariesResolution { - // Since we are using the cached libraries resolution - // the library manager is not needed. - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm - } else if ctx.LibrariesManager == nil { - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm - - builtInLibrariesFolders := ctx.BuiltInLibrariesDirs - if builtInLibrariesFolders != nil { - if err := builtInLibrariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) - } - lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) - } - - if ctx.ActualPlatform != ctx.TargetPlatform { - lm.AddPlatformReleaseLibrariesDir(ctx.ActualPlatform, libraries.ReferencedPlatformBuiltIn) - } - lm.AddPlatformReleaseLibrariesDir(ctx.TargetPlatform, libraries.PlatformBuiltIn) - - librariesFolders := ctx.OtherLibrariesDirs - if err := librariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) - } - for _, folder := range librariesFolders { - lm.AddLibrariesDir(folder, libraries.User) - } - - for _, status := range lm.RescanLibraries() { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. - if ctx.Verbose { - ctx.Warn(status.Message()) - } - } - - for _, dir := range ctx.LibraryDirs { - // Libraries specified this way have top priority - if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return err - } - } - } - - resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) - } - if err := resolver.ScanUserAndUnmanagedLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) - } - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.TargetPlatform); err != nil { - return errors.WithStack(err) - } - if ctx.ActualPlatform != ctx.TargetPlatform { - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.ActualPlatform); err != nil { - return errors.WithStack(err) - } - } - ctx.LibrariesResolver = resolver - - return nil -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 5c94573f349..0b629365eba 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -22,13 +22,13 @@ import ( "os" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -80,7 +80,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } - includes = f.Map(includes, utils.WrapWithHyphenI) + includes = f.Map(includes, cpp.WrapWithHyphenI) var err error diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 5ee81bb4385..832bc9b0f8f 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,12 +18,12 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -37,8 +37,9 @@ type LibrariesBuilder struct{} func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) - libs := ctx.ImportedLibraries + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) + libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) @@ -66,9 +67,9 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -77,7 +78,7 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib } } } - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FLOAT_ABI_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -200,7 +201,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } } else { if library.UtilityDir != nil { - includes = append(includes, utils.WrapWithHyphenI(library.UtilityDir.String())) + includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } libObjectFiles, err := builder_utils.CompileFiles(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) if err != nil { diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index cc7c293199d..1f324576586 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -93,7 +93,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return errors.WithStack(err) } @@ -114,10 +114,10 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err - } +} _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) return err diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index d466aca2b06..c064fbe717b 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -50,7 +50,7 @@ func (s *Sizer) Run(ctx *types.Context) error { } func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -179,7 +179,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 300cb6836af..51057a21835 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,10 +16,10 @@ package phases import ( + "github.com/arduino/arduino-cli/arduino/builder/cpp" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" ) @@ -28,7 +28,8 @@ type SketchBuilder struct{} func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go deleted file mode 100644 index 7dff77c7dde..00000000000 --- a/legacy/builder/preprocess_sketch.go +++ /dev/null @@ -1,89 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - - bldr "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - properties "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" -) - -func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { - if err := ctx.BuildPath.Join("preproc").MkdirAll(); err != nil { - return errors.WithStack(err) - } - - sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - targetFile := ctx.BuildPath.Join("preproc", "sketch_merged.cpp") - gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(gccStdout) - ctx.WriteStderr(gccStderr) - } - if err != nil { - return err - } - - buildProperties := properties.NewMap() - buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") - buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") - buildProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) - buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") - buildProperties.Merge(ctx.BuildProperties) - buildProperties.Merge(buildProperties.SubTree("tools").SubTree("arduino-preprocessor")) - buildProperties.SetPath("source_file", targetFile) - - pattern := buildProperties.Get("pattern") - if pattern == "" { - return errors.New(tr("arduino-preprocessor pattern is missing")) - } - - commandLine := buildProperties.ExpandPropsInString(pattern) - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return errors.WithStack(err) - } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...) - - if runtime.GOOS == "windows" { - // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) - command.Dir = filepath.VolumeName(command.Args[0]) + "/" - //command.Args[0], _ = filepath.Rel(command.Dir, command.Args[0]) - } - - verbose := ctx.Verbose - if verbose { - fmt.Println(commandLine) - } - - buf, err := command.Output() - if err != nil { - return errors.New(errors.WithStack(err).Error() + string(err.(*exec.ExitError).Stderr)) - } - - result := utils.NormalizeUTF8(buf) - return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, result, ctx.SketchBuildPath) -} diff --git a/legacy/builder/print_used_and_not_used_libraries.go b/legacy/builder/print_used_and_not_used_libraries.go deleted file mode 100644 index 971036bea45..00000000000 --- a/legacy/builder/print_used_and_not_used_libraries.go +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type PrintUsedAndNotUsedLibraries struct { - // Was there an error while compiling the sketch? - SketchError bool -} - -func (s *PrintUsedAndNotUsedLibraries) Run(ctx *types.Context) error { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !s.SketchError && !ctx.Verbose { - return nil - } - - res := "" - for header, libResResult := range ctx.LibrariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if s.SketchError { - ctx.Warn(res) - } else { - ctx.Info(res) - } - time.Sleep(100 * time.Millisecond) - return nil -} diff --git a/legacy/builder/print_used_libraries_if_verbose.go b/legacy/builder/print_used_libraries_if_verbose.go index 8c32c925995..15211a46519 100644 --- a/legacy/builder/print_used_libraries_if_verbose.go +++ b/legacy/builder/print_used_libraries_if_verbose.go @@ -24,11 +24,11 @@ import ( type PrintUsedLibrariesIfVerbose struct{} func (s *PrintUsedLibrariesIfVerbose) Run(ctx *types.Context) error { - if !ctx.Verbose || len(ctx.ImportedLibraries) == 0 { + if !ctx.Verbose || len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { return nil } - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { legacy := "" if library.IsLegacy { legacy = tr("(legacy)") diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index a6b552b2c6d..01257267287 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -44,14 +44,14 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } if ctx.OnlyUpdateCompilationDatabase && s.SkipIfOnlyUpdatingCompilationDatabase { if ctx.Verbose { - ctx.Info(tr("Skipping: %[1]s", strings.Join(command.Args, " "))) + ctx.Info(tr("Skipping: %[1]s", strings.Join(command.GetArgs(), " "))) } return nil } diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go deleted file mode 100644 index fc1edb5acb2..00000000000 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "testing" - - "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" - "github.com/stretchr/testify/require" -) - -func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { - ctx := &types.Context{} - - command := builder.AddAdditionalEntriesToContext{} - NoError(t, command.Run(ctx)) - - require.Empty(t, ctx.SketchBuildPath) - require.Empty(t, ctx.LibrariesBuildPath) - require.Empty(t, ctx.CoreBuildPath) - - require.NotNil(t, ctx.WarningsLevel) - - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) -} - -func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { - ctx := &types.Context{} - ctx.BuildPath = paths.New("folder") - - command := builder.AddAdditionalEntriesToContext{} - NoError(t, command.Run(ctx)) - - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), ctx.SketchBuildPath) - require.Equal(t, Abs(t, paths.New("folder", "libraries")), ctx.LibrariesBuildPath) - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), ctx.CoreBuildPath) - - require.NotNil(t, ctx.WarningsLevel) - - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 58a372d6e41..696bcfad957 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -22,6 +22,7 @@ import ( "time" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder" @@ -39,9 +40,18 @@ func cleanUpBuilderTestContext(t *testing.T, ctx *types.Context) { } } -func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string) *types.Context { +type skipContextPreparationStepName string + +const skipLibraries = skipContextPreparationStepName("libraries") + +func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string, skips ...skipContextPreparationStepName) *types.Context { DownloadCoresAndToolsAndLibraries(t) + stepToSkip := map[skipContextPreparationStepName]bool{} + for _, skip := range skips { + stepToSkip[skip] = true + } + if ctx == nil { ctx = &types.Context{} } @@ -57,6 +67,18 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.BuildPath = buildPath } + buildPath := ctx.BuildPath + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + NoError(t, err) + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + NoError(t, err) + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + NoError(t, err) + + ctx.SketchBuildPath = sketchBuildPath + ctx.LibrariesBuildPath = librariesBuildPath + ctx.CoreBuildPath = coreBuildPath + // Create a Package Manager from the given context // This should happen only on legacy arduino-builder. // Hopefully this piece will be removed once the legacy package will be cleanedup. @@ -81,6 +103,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.Sketch = sk } + ctx.Builder = bldr.NewBuilder(ctx.Sketch) if fqbn != "" { ctx.FQBN = parseFQBN(t, fqbn) targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := pme.ResolveFQBN(ctx.FQBN) @@ -88,7 +111,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat requiredTools, err := pme.FindToolsRequiredForBuild(targetPlatform, buildPlatform) require.NoError(t, err) - buildProperties = bldr.SetupBuildProperties(buildProperties, ctx.BuildPath, ctx.Sketch, false /*OptimizeForDebug*/) + buildProperties = ctx.Builder.SetupBuildProperties(buildProperties, ctx.BuildPath, false /*OptimizeForDebug*/) ctx.PackageManager = pme ctx.TargetBoard = targetBoard ctx.BuildProperties = buildProperties @@ -98,6 +121,30 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.RequiredTools = requiredTools } + if ctx.Sketch != nil { + require.False(t, ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical())) + } + + if !stepToSkip[skipLibraries] { + lm, libsResolver, _, err := detector.LibrariesLoader( + false, nil, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) + + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( + lm, libsResolver, + ctx.Verbose, + false, + false, + func(msg string) { ctx.Info(msg) }, + func(msg string) { ctx.Warn(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + ) + } + return ctx } diff --git a/legacy/builder/test/builder_utils_test.go b/legacy/builder/test/builder_utils_test.go index b20e7e2555a..f032eabeddc 100644 --- a/legacy/builder/test/builder_utils_test.go +++ b/legacy/builder/test/builder_utils_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -42,7 +42,7 @@ func TestObjFileIsUpToDateObjMissing(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, nil, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, nil, nil) NoError(t, err) require.False(t, upToDate) } @@ -54,7 +54,7 @@ func TestObjFileIsUpToDateDepMissing(t *testing.T) { objFile := tempFile(t, "obj") defer objFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, nil) NoError(t, err) require.False(t, upToDate) } @@ -70,7 +70,7 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -86,7 +86,7 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer depFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -110,7 +110,7 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { data := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(data)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -132,7 +132,7 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { res := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -156,7 +156,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { res := sourceFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } diff --git a/legacy/builder/test/hardware_loader_test.go b/legacy/builder/test/hardware_loader_test.go index f900ad38ac3..ae87e1c41c7 100644 --- a/legacy/builder/test/hardware_loader_test.go +++ b/legacy/builder/test/hardware_loader_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -34,7 +33,8 @@ func TestLoadHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware")), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -66,17 +66,9 @@ func TestLoadHardwareMixingUserHardwareFolder(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - packages := ctx.PackageManager.GetPackages() if runtime.GOOS == "windows" { @@ -134,7 +126,7 @@ func TestLoadHardwareWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -175,7 +167,7 @@ func TestLoadLotsOfHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff", "downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() diff --git a/legacy/builder/test/includes_finder_with_regexp_test.go b/legacy/builder/test/includes_finder_with_regexp_test.go index 3a55415aa88..16dd07da48f 100644 --- a/legacy/builder/test/includes_finder_with_regexp_test.go +++ b/legacy/builder/test/includes_finder_with_regexp_test.go @@ -18,7 +18,7 @@ package test import ( "testing" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/stretchr/testify/require" ) @@ -27,13 +27,13 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { - include := builder.IncludesFinderWithRegExp("") + include := detector.IncludesFinderWithRegExp("") require.Equal(t, "", include) } @@ -43,7 +43,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -53,7 +53,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -62,7 +62,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } @@ -71,7 +71,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "register.h", include) } diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 143c1866221..e4e1cacfa59 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -20,17 +20,17 @@ import ( "sort" "testing" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) -func extractLibraries(ctx *types.Context) []*libraries.Library { +func extractLibraries(libs map[string]libraries.List) []*libraries.Library { res := []*libraries.Library{} - for _, lib := range ctx.LibrariesManager.Libraries { + for _, lib := range libs { for _, libAlternative := range lib { res = append(res, libAlternative) } @@ -47,22 +47,20 @@ func TestLoadLibrariesAVR(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := detector.LibrariesLoader( + false, nil, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libs := extractLibraries(ctx) + libs := extractLibraries(lm.Libraries) require.Equal(t, 24, len(libs)) sort.Sort(ByLibraryName(libs)) @@ -125,21 +123,21 @@ func TestLoadLibrariesAVR(t *testing.T) { idx++ require.Equal(t, "Wire", libs[idx].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs = libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Adafruit_PN532.h") + libs = libsResolver.AlternativesFor("Adafruit_PN532.h") require.Len(t, libs, 1) require.Equal(t, "Adafruit PN532", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -153,22 +151,20 @@ func TestLoadLibrariesSAM(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := detector.LibrariesLoader( + false, nil, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "sam", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libraries := extractLibraries(ctx) + libraries := extractLibraries(lm.Libraries) require.Equal(t, 22, len(libraries)) sort.Sort(ByLibraryName(libraries)) @@ -208,17 +204,17 @@ func TestLoadLibrariesSAM(t *testing.T) { idx++ require.Equal(t, "Wire", libraries[idx].Name) - libs := ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs := libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -232,16 +228,14 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := detector.LibrariesLoader( + false, nil, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) @@ -257,16 +251,14 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := detector.LibrariesLoader( + false, nil, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 4, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) diff --git a/legacy/builder/test/merge_sketch_with_bootloader_test.go b/legacy/builder/test/merge_sketch_with_bootloader_test.go index b36102f6c40..43387eeaa8d 100644 --- a/legacy/builder/test/merge_sketch_with_bootloader_test.go +++ b/legacy/builder/test/merge_sketch_with_bootloader_test.go @@ -70,7 +70,6 @@ func TestMergeSketchWithBootloader(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -129,7 +128,6 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -152,15 +150,6 @@ func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) buildPath := ctx.BuildPath - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) @@ -222,7 +211,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } diff --git a/legacy/builder/test/recipe_runner_test.go b/legacy/builder/test/recipe_runner_test.go index 6fb90bf7e68..5d9a53ef68e 100644 --- a/legacy/builder/test/recipe_runner_test.go +++ b/legacy/builder/test/recipe_runner_test.go @@ -35,7 +35,6 @@ func TestRecipeRunner(t *testing.T) { buildProperties.Set("recipe.hooks.prebuild.1.pattern", "echo") commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, } diff --git a/legacy/builder/test/setup_build_properties_test.go b/legacy/builder/test/setup_build_properties_test.go index 1b595d5d69c..2c0344812a2 100644 --- a/legacy/builder/test/setup_build_properties_test.go +++ b/legacy/builder/test/setup_build_properties_test.go @@ -19,7 +19,6 @@ import ( "path/filepath" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -34,14 +33,6 @@ func TestSetupBuildProperties(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) @@ -108,14 +99,6 @@ func TestSetupBuildPropertiesUserHardware(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) @@ -134,15 +117,6 @@ func TestSetupBuildPropertiesWithMissingPropsFromParentPlatformTxtFiles(t *testi ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) diff --git a/legacy/builder/test/tools_loader_test.go b/legacy/builder/test/tools_loader_test.go index 7138df09afc..e28f828401d 100644 --- a/legacy/builder/test/tools_loader_test.go +++ b/legacy/builder/test/tools_loader_test.go @@ -66,7 +66,7 @@ func TestLoadTools(t *testing.T) { HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -107,7 +107,7 @@ func TestLoadToolsWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -131,7 +131,7 @@ func TestLoadLotsOfTools(t *testing.T) { HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index f6965cee0ff..24001e674ab 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -18,6 +18,7 @@ package test import ( "testing" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -36,7 +37,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( + nil, nil, false, false, false, nil, nil, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) @@ -56,7 +60,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( + nil, nil, false, false, false, nil, nil, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err := cmd.Run(ctx) @@ -73,8 +80,10 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { NoError(t, temp.Join("dummy_file").WriteFile([]byte{})) ctx := &types.Context{} + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( + nil, nil, false, false, false, nil, nil, nil, nil, + ) ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{} cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0a160324a46..a6570bf3268 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -23,11 +23,9 @@ import ( "sync" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -64,6 +62,9 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { + Builder *builder.Builder + SketchLibrariesDetector *detector.SketchLibrariesDetector + // Build options HardwareDirs paths.PathList BuiltInToolsDirs paths.PathList @@ -100,14 +101,6 @@ type Context struct { Sketch *sketch.Sketch WarningsLevel string - // Libraries handling - LibrariesManager *librariesmanager.LibrariesManager - LibrariesResolver *librariesresolver.Cpp - ImportedLibraries libraries.List - LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeFolders paths.PathList - UseCachedLibrariesResolution bool - // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 2ebf92bae5a..8678fd2fe51 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -15,91 +15,6 @@ package types -import ( - "fmt" - - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/sketch" - paths "github.com/arduino/go-paths-helper" -) - -type SourceFile struct { - // Path to the source file within the sketch/library root folder - relativePath *paths.Path - - // ExtraIncludePath contains an extra include path that must be - // used to compile this source file. - // This is mainly used for source files that comes from old-style libraries - // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. - extraIncludePath *paths.Path - - // The source root for the given origin, where its source files - // can be found. Prepending this to SourceFile.RelativePath will give - // the full path to that source file. - sourceRoot *paths.Path - - // The build root for the given origin, where build products will - // be placed. Any directories inside SourceFile.RelativePath will be - // appended here. - buildRoot *paths.Path -} - -func (f *SourceFile) Equals(g *SourceFile) bool { - return f.relativePath.EqualsTo(g.relativePath) && - f.buildRoot.EqualsTo(g.buildRoot) && - f.sourceRoot.EqualsTo(g.sourceRoot) -} - -// Create a SourceFile containing the given source file path within the -// given origin. The given path can be absolute, or relative within the -// origin's root source folder -func MakeSourceFile(ctx *Context, origin interface{}, path *paths.Path) (*SourceFile, error) { - res := &SourceFile{} - - switch o := origin.(type) { - case *sketch.Sketch: - res.buildRoot = ctx.SketchBuildPath - res.sourceRoot = ctx.SketchBuildPath - case *libraries.Library: - res.buildRoot = ctx.LibrariesBuildPath.Join(o.DirName) - res.sourceRoot = o.SourceDir - res.extraIncludePath = o.UtilityDir - default: - panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) - } - - if path.IsAbs() { - var err error - path, err = res.sourceRoot.RelTo(path) - if err != nil { - return nil, err - } - } - res.relativePath = path - return res, nil -} - -func (f *SourceFile) ExtraIncludePath() *paths.Path { - return f.extraIncludePath -} - -func (f *SourceFile) SourcePath() *paths.Path { - return f.sourceRoot.JoinPath(f.relativePath) -} - -func (f *SourceFile) ObjectPath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".o") -} - -func (f *SourceFile) DepfilePath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".d") -} - -type LibraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - type Command interface { Run(ctx *Context) error } diff --git a/legacy/builder/unused_compiled_libraries_remover.go b/legacy/builder/unused_compiled_libraries_remover.go index d1b36b147ac..03ea5b85d01 100644 --- a/legacy/builder/unused_compiled_libraries_remover.go +++ b/legacy/builder/unused_compiled_libraries_remover.go @@ -31,7 +31,7 @@ func (s *UnusedCompiledLibrariesRemover) Run(ctx *types.Context) error { return nil } - libraryNames := toLibraryNames(ctx.ImportedLibraries) + libraryNames := toLibraryNames(ctx.SketchLibrariesDetector.ImportedLibraries()) files, err := librariesBuildPath.ReadDir() if err != nil { diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 102b007883f..ec34d1f2c3b 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -18,44 +18,14 @@ package utils import ( "bytes" "os" - "os/exec" "strings" - "unicode" + "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/text/runes" - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" ) -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} - -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") - -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] -} - -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { - // See if the file is readable by opening it - f, err := file.Open() - if err != nil { - return false - } - f.Close() - return true -} - -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.Replace(arg, "\\", "\\\\", -1) @@ -81,30 +51,30 @@ const ( Capture = 3 // Capture into buffer ) -func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { +func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) { if ctx.Verbose { - ctx.Info(PrintableCommand(command.Args)) + ctx.Info(PrintableCommand(command.GetArgs())) } + stdoutBuffer := &bytes.Buffer{} if stdout == Capture { - buffer := &bytes.Buffer{} - command.Stdout = buffer + command.RedirectStdoutTo(stdoutBuffer) } else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { if ctx.Stdout != nil { - command.Stdout = ctx.Stdout + command.RedirectStdoutTo(ctx.Stdout) } else { - command.Stdout = os.Stdout + command.RedirectStdoutTo(os.Stdout) } } + stderrBuffer := &bytes.Buffer{} if stderr == Capture { - buffer := &bytes.Buffer{} - command.Stderr = buffer + command.RedirectStderrTo(stderrBuffer) } else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { if ctx.Stderr != nil { - command.Stderr = ctx.Stderr + command.RedirectStderrTo(ctx.Stderr) } else { - command.Stderr = os.Stderr + command.RedirectStderrTo(os.Stderr) } } @@ -114,39 +84,7 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) } err = command.Wait() - - var outbytes, errbytes []byte - if buf, ok := command.Stdout.(*bytes.Buffer); ok { - outbytes = buf.Bytes() - } - if buf, ok := command.Stderr.(*bytes.Buffer); ok { - errbytes = buf.Bytes() - } - - return outbytes, errbytes, errors.WithStack(err) -} - -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { - fileFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - paths.FilterOutDirectories(), - FilterReadableFiles, - ) - if len(extensions) > 0 { - fileFilter = paths.AndFilter( - paths.FilterSuffixes(extensions...), - fileFilter, - ) - } - if recurse { - dirFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - ) - return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) - } - return dir.ReadDir(fileFilter) + return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } type loggerAction struct { @@ -169,11 +107,3 @@ func (l *loggerAction) Run(ctx *types.Context) error { func LogIfVerbose(warn bool, msg string) types.Command { return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} } - -// Normalizes an UTF8 byte slice -// TODO: use it more often troughout all the project (maybe on logger interface?) -func NormalizeUTF8(buf []byte) []byte { - t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) - result, _, _ := transform.Bytes(t, buf) - return result -} diff --git a/legacy/builder/warn_about_arch_incompatible_libraries.go b/legacy/builder/warn_about_arch_incompatible_libraries.go index 712f79b2fe5..c815a54f3aa 100644 --- a/legacy/builder/warn_about_arch_incompatible_libraries.go +++ b/legacy/builder/warn_about_arch_incompatible_libraries.go @@ -33,7 +33,7 @@ func (s *WarnAboutArchIncompatibleLibraries) Run(ctx *types.Context) error { archs = append(archs, strings.Split(overrides, ",")...) } - for _, importedLibrary := range ctx.ImportedLibraries { + for _, importedLibrary := range ctx.SketchLibrariesDetector.ImportedLibraries() { if !importedLibrary.SupportsAnyArchitectureIn(archs...) { ctx.Info( tr("WARNING: library %[1]s claims to run on %[2]s architecture(s) and may be incompatible with your current board which runs on %[3]s architecture(s).",