From d97cac78848a82418b96ce9badb4abd0aab06f0e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Sat, 2 Sep 2023 21:21:39 +0200 Subject: [PATCH] refactor find_includes --- arduino/builder/detector/detector.go | 795 ++++++++++++++++++ arduino/builder/libraries.go | 240 ------ .../preprocessor/arduino_preprocessor.go | 2 +- arduino/builder/preprocessor/gcc.go | 2 +- arduino/builder/utils/utils.go | 185 ++++ commands/compile/compile.go | 8 +- legacy/builder/builder.go | 22 +- legacy/builder/builder_utils/utils.go | 123 +-- legacy/builder/container_find_includes.go | 474 ----------- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/includes_finder_with_regexp.go | 44 - legacy/builder/phases/core_builder.go | 2 +- legacy/builder/phases/libraries_builder.go | 5 +- legacy/builder/phases/sketch_builder.go | 5 +- legacy/builder/test/builder_test.go | 3 + .../unused_compiled_libraries_remover_test.go | 6 +- legacy/builder/types/accessories.go | 41 - legacy/builder/types/context.go | 6 +- legacy/builder/types/types.go | 80 -- legacy/builder/utils/utils.go | 61 -- 20 files changed, 1027 insertions(+), 1079 deletions(-) create mode 100644 arduino/builder/detector/detector.go delete mode 100644 arduino/builder/libraries.go create mode 100644 arduino/builder/utils/utils.go delete mode 100644 legacy/builder/container_find_includes.go delete mode 100644 legacy/builder/includes_finder_with_regexp.go delete mode 100644 legacy/builder/types/accessories.go diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go new file mode 100644 index 00000000000..aa00a02e718 --- /dev/null +++ b/arduino/builder/detector/detector.go @@ -0,0 +1,795 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "golang.org/x/exp/slices" + + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/arduino/libraries" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +var tr = i18n.Tr + +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + onlyUpdateCompilationDatabase bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + verboseStdoutFn func(data []byte) + verboseStderrFn func(data []byte) + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult + includeFolders paths.PathList +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + verbose, useCachedLibrariesResolution bool, + onlyUpdateCompilationDatabase bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), + verboseStdoutFn func(data []byte), + verboseStderrFn func(data []byte), +) *SketchLibrariesDetector { + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + verboseStdoutFn: verboseStdoutFn, + verboseStderrFn: verboseStderrFn, + importedLibraries: libraries.List{}, + includeFolders: paths.PathList{}, + onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) resolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +// ImportedLibraries todo +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// UseCachedLibrariesResolution todo +func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { + return l.useCachedLibrariesResolution +} + +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + +func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { + // TODO should we do a deep copy? + return l.includeFolders +} + +// AppendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +// Original comment: +// Append the given folder to the include path and match or append it to +// the cache. sourceFilePath and include indicate the source of this +// include (e.g. what #include line in what file it was resolved from) +// and should be the empty string for the default include folders, like +// the core or variant. +func (l *SketchLibrariesDetector) AppendIncludeFolder( + cache *includeCache, + sourceFilePath *paths.Path, + include string, + folder *paths.Path, +) { + l.includeFolders = append(l.includeFolders, folder) + cache.ExpectEntry(sourceFilePath, include, folder) +} + +// FindIncludes todo +func (l *SketchLibrariesDetector) FindIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + err := l.findIncludes(buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch) + if err != nil && l.onlyUpdateCompilationDatabase { + l.verboseInfoFn( + fmt.Sprintf( + "%s: %s", + tr("An error occurred detecting libraries"), + tr("the compilation database may be incomplete or inaccurate"), + ), + ) + return nil + } + return err +} + +func (l *SketchLibrariesDetector) findIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + librariesResolutionCache := buildPath.Join("libraries.cache") + if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { + if d, err := librariesResolutionCache.ReadFile(); err != nil { + return err + } else { + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { + return err + } + } + if l.verbose { + l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) + } + return nil + } + + cachePath := buildPath.Join("includes.cache") + cache := readCache(cachePath) + + l.AppendIncludeFolder(cache, nil, "", buildCorePath) + if buildVariantPath != nil { + l.AppendIncludeFolder(cache, nil, "", buildVariantPath) + } + + sourceFileQueue := &UniqueSourceFileQueue{} + + if !l.useCachedLibrariesResolution { + sketch := sketch + mergedfile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(mergedfile) + + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) + srcSubfolderPath := sketchBuildPath.Join("src") + if srcSubfolderPath.IsDir() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) + } + + for !sourceFileQueue.Empty() { + err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) + if err != nil { + cachePath.Remove() + return errors.WithStack(err) + } + } + + // Finalize the cache + cache.ExpectEnd() + if err := writeCache(cache, cachePath); err != nil { + return errors.WithStack(err) + } + } + + if err := l.failIfImportedLibraryIsWrong(); err != nil { + return errors.WithStack(err) + } + + if d, err := json.Marshal(l.includeFolders); err != nil { + return err + } else if err := librariesResolutionCache.WriteFile(d); err != nil { + return err + } + + return nil +} + +func (l *SketchLibrariesDetector) findIncludesUntilDone( + cache *includeCache, + sourceFileQueue *UniqueSourceFileQueue, + buildProperties *properties.Map, + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + platformArch string, +) error { + sourceFile := sourceFileQueue.Pop() + sourcePath := sourceFile.SourcePath() + targetFilePath := paths.NullPath() + depPath := sourceFile.DepfilePath() + objPath := sourceFile.ObjectPath() + + // TODO: This should perhaps also compare against the + // include.cache file timestamp. Now, it only checks if the file + // changed after the object file was generated, but if it + // changed between generating the cache and the object file, + // this could show the file as unchanged when it really is + // changed. Changing files during a build isn't really + // supported, but any problems from it should at least be + // resolved when doing another build, which is not currently the + // case. + // TODO: This reads the dependency file, but the actual building + // does it again. Should the result be somehow cached? Perhaps + // remove the object file if it is found to be stale? + unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) + if err != nil { + return errors.WithStack(err) + } + + first := true + for { + cache.ExpectFile(sourcePath) + + // Libraries may require the "utility" directory to be added to the include + // search path, but only for the source code of the library, so we temporary + // copy the current search path list and add the library' utility directory + // if needed. + includeFolders := l.includeFolders + if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { + includeFolders = append(includeFolders, extraInclude) + } + + var preprocErr error + var preprocStderr []byte + + var missingIncludeH string + if unchanged && cache.valid { + missingIncludeH = cache.Next().Include + if first && l.verbose { + l.verboseInfoFn(tr("Using cached library dependencies for file: %[1]s", sourcePath)) + } + } else { + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + // Unwrap error and see if it is an ExitError. + if preprocErr == nil { + // Preprocessor successful, done + missingIncludeH = "" + } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { + // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors + return errors.WithStack(preprocErr) + } else { + missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) + if missingIncludeH == "" && l.verbose { + l.verboseInfoFn(tr("Error while detecting libraries included by %[1]s", sourcePath)) + } + } + } + + if missingIncludeH == "" { + // No missing includes found, we're done + cache.ExpectEntry(sourcePath, "", nil) + return nil + } + + library := l.resolveLibrary(missingIncludeH, platformArch) + if library == nil { + // Library could not be resolved, show error + if preprocErr == nil || preprocStderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + if preprocErr == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New(tr("Internal error in cache")) + } + } + l.verboseStderrFn(preprocStderr) + return errors.WithStack(preprocErr) + } + + // Add this library to the list of libraries, the + // include path and queue its source files for further + // include scanning + l.AppendImportedLibraries(library) + l.AppendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + + if library.Precompiled && library.PrecompiledWithSources { + // Fully precompiled libraries should have no dependencies to avoid ABI breakage + if l.verbose { + l.verboseInfoFn(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) + } + } else { + for _, sourceDir := range library.SourceDirs() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) + } + } + first = false + } +} + +func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + sourceFileQueue *UniqueSourceFileQueue, + origin interface{}, + folder *paths.Path, + recurse bool, +) error { + sourceFileExtensions := []string{} + for k := range globals.SourceFilesValidExtensions { + sourceFileExtensions = append(sourceFileExtensions, k) + } + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) + if err != nil { + return errors.WithStack(err) + } + + for _, filePath := range filePaths { + sourceFile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(sourceFile) + } + + return nil +} + +func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { + if len(l.importedLibraries) == 0 { + return nil + } + + for _, library := range l.importedLibraries { + if !library.IsLegacy { + if library.InstallDir.Join("arch").IsDir() { + return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) + } + for _, propName := range libraries.MandatoryProperties { + if !library.Properties.ContainsKey(propName) { + return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) + } + } + if library.Layout == libraries.RecursiveLayout { + if library.UtilityDir != nil { + return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) + } + } + } + } + + return nil +} + +var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") + +func IncludesFinderWithRegExp(source string) string { + match := INCLUDE_REGEXP.FindStringSubmatch(source) + if match != nil { + return strings.TrimSpace(match[1]) + } + return findIncludeForOldCompilers(source) +} + +func findIncludeForOldCompilers(source string) string { + lines := strings.Split(source, "\n") + for _, line := range lines { + splittedLine := strings.Split(line, ":") + for i := range splittedLine { + if strings.Contains(splittedLine[i], "fatal error") { + return strings.TrimSpace(splittedLine[i+1]) + } + } + } + return "" +} + +type SourceFile struct { + // Path to the source file within the sketch/library root folder + relativePath *paths.Path + + // ExtraIncludePath contains an extra include path that must be + // used to compile this source file. + // This is mainly used for source files that comes from old-style libraries + // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. + extraIncludePath *paths.Path + + // The source root for the given origin, where its source files + // can be found. Prepending this to SourceFile.RelativePath will give + // the full path to that source file. + sourceRoot *paths.Path + + // The build root for the given origin, where build products will + // be placed. Any directories inside SourceFile.RelativePath will be + // appended here. + buildRoot *paths.Path +} + +func (f *SourceFile) Equals(g *SourceFile) bool { + return f.relativePath.EqualsTo(g.relativePath) && + f.buildRoot.EqualsTo(g.buildRoot) && + f.sourceRoot.EqualsTo(g.sourceRoot) +} + +// Create a SourceFile containing the given source file path within the +// given origin. The given path can be absolute, or relative within the +// origin's root source folder +func MakeSourceFile( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + origin interface{}, + path *paths.Path, +) (*SourceFile, error) { + res := &SourceFile{} + + switch o := origin.(type) { + case *sketch.Sketch: + res.buildRoot = sketchBuildPath + res.sourceRoot = sketchBuildPath + case *libraries.Library: + res.buildRoot = librariesBuildPath.Join(o.DirName) + res.sourceRoot = o.SourceDir + res.extraIncludePath = o.UtilityDir + default: + panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) + } + + if path.IsAbs() { + var err error + path, err = res.sourceRoot.RelTo(path) + if err != nil { + return nil, err + } + } + res.relativePath = path + return res, nil +} + +func (f *SourceFile) ExtraIncludePath() *paths.Path { + return f.extraIncludePath +} + +func (f *SourceFile) SourcePath() *paths.Path { + return f.sourceRoot.JoinPath(f.relativePath) +} + +func (f *SourceFile) ObjectPath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".o") +} + +func (f *SourceFile) DepfilePath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".d") +} + +// LibrariesLoader todo +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { + // Since we are using the cached libraries resolution + // the library manager is not needed. + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) + + builtInLibrariesFolders := builtInLibrariesDirs + if builtInLibrariesFolders != nil { + if err := builtInLibrariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) + } + + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) + } + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) + + librariesFolders := otherLibrariesDirs + if err := librariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + for _, folder := range librariesFolders { + lm.AddLibrariesDir(folder, libraries.User) + } + + for _, status := range lm.RescanLibraries() { + // With the refactoring of the initialization step of the CLI we changed how + // errors are returned when loading platforms and libraries, that meant returning a list of + // errors instead of a single one to enhance the experience for the user. + // I have no intention right now to start a refactoring of the legacy package too, so + // here's this shitty solution for now. + // When we're gonna refactor the legacy package this will be gone. + verboseOut.Write([]byte(status.Message())) + } + + for _, dir := range libraryDirs { + // Libraries specified this way have top priority + if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + } + + resolver := librariesresolver.NewCppResolver() + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + return lm, resolver, verboseOut.Bytes(), nil +} + +func (l *SketchLibrariesDetector) OnlyUpdateCompilationDatabase() bool { + return l.onlyUpdateCompilationDatabase +} + +type includeCacheEntry struct { + Sourcefile *paths.Path + Include string + Includepath *paths.Path +} + +func (entry *includeCacheEntry) String() string { + return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", + entry.Sourcefile, entry.Include, entry.Includepath) +} + +func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { + return entry.String() == other.String() +} + +type includeCache struct { + // Are the cache contents valid so far? + valid bool + // Index into entries of the next entry to be processed. Unused + // when the cache is invalid. + next int + entries []*includeCacheEntry +} + +// Return the next cache entry. Should only be called when the cache is +// valid and a next entry is available (the latter can be checked with +// ExpectFile). Does not advance the cache. +func (cache *includeCache) Next() *includeCacheEntry { + return cache.entries[cache.next] +} + +// Check that the next cache entry is about the given file. If it is +// not, or no entry is available, the cache is invalidated. Does not +// advance the cache. +func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { + if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Check that the next entry matches the given values. If so, advance +// the cache. If not, the cache is invalidated. If the cache is +// invalidated, or was already invalid, an entry with the given values +// is appended. +func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { + entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} + if cache.valid { + if cache.next < len(cache.entries) && cache.Next().Equals(entry) { + cache.next++ + } else { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } + } + + if !cache.valid { + cache.entries = append(cache.entries, entry) + } +} + +// Check that the cache is completely consumed. If not, the cache is +// invalidated. +func (cache *includeCache) ExpectEnd() { + if cache.valid && cache.next < len(cache.entries) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Read the cache from the given file +func readCache(path *paths.Path) *includeCache { + bytes, err := path.ReadFile() + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result := &includeCache{} + err = json.Unmarshal(bytes, &result.entries) + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result.valid = true + return result +} + +// Write the given cache to the given file if it is invalidated. If the +// cache is still valid, just update the timestamps of the file. +func writeCache(cache *includeCache, path *paths.Path) error { + // If the cache was still valid all the way, just touch its file + // (in case any source file changed without influencing the + // includes). If it was invalidated, overwrite the cache with + // the new contents. + if cache.valid { + path.Chtimes(time.Now(), time.Now()) + } else { + bytes, err := json.MarshalIndent(cache.entries, "", " ") + if err != nil { + return errors.WithStack(err) + } + err = path.WriteFile(bytes) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +type UniqueSourceFileQueue []*SourceFile + +func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { + if !queue.Contains(value) { + *queue = append(*queue, value) + } +} + +func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { + return slices.ContainsFunc(queue, target.Equals) +} + +func (queue *UniqueSourceFileQueue) Pop() *SourceFile { + old := *queue + x := old[0] + *queue = old[1:] + return x +} + +func (queue UniqueSourceFileQueue) Empty() bool { + return len(queue) == 0 +} diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go deleted file mode 100644 index 2f8219c5241..00000000000 --- a/arduino/builder/libraries.go +++ /dev/null @@ -1,240 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/arduino/cores" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type libraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - -// SketchLibrariesDetector todo -type SketchLibrariesDetector struct { - librariesManager *librariesmanager.LibrariesManager - librariesResolver *librariesresolver.Cpp - useCachedLibrariesResolution bool - verbose bool - verboseInfoFn func(msg string) - verboseWarnFn func(msg string) - importedLibraries libraries.List - librariesResolutionResults map[string]libraryResolutionResult -} - -// NewSketchLibrariesDetector todo -func NewSketchLibrariesDetector( - lm *librariesmanager.LibrariesManager, - libsResolver *librariesresolver.Cpp, - verbose, useCachedLibrariesResolution bool, - verboseInfoFn func(msg string), - verboseWarnFn func(msg string), -) *SketchLibrariesDetector { - return &SketchLibrariesDetector{ - librariesManager: lm, - librariesResolver: libsResolver, - useCachedLibrariesResolution: useCachedLibrariesResolution, - librariesResolutionResults: map[string]libraryResolutionResult{}, - verbose: verbose, - verboseInfoFn: verboseInfoFn, - verboseWarnFn: verboseWarnFn, - importedLibraries: libraries.List{}, - } -} - -// ResolveLibrary todo -func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *libraries.Library { - importedLibraries := l.importedLibraries - candidates := l.librariesResolver.AlternativesFor(header) - - if l.verbose { - l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) - l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := l.librariesResolver.ResolveFor(header, platformArch) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - l.librariesResolutionResults[header] = libraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected -} - -// ImportedLibraries todo -func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { - // TODO understand if we have to do a deepcopy - return l.importedLibraries -} - -// AppendImportedLibraries todo should rename this, probably after refactoring the -// container_find_includes command. -func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { - l.importedLibraries = append(l.importedLibraries, library) -} - -// UseCachedLibrariesResolution todo -func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { - return l.useCachedLibrariesResolution -} - -// PrintUsedAndNotUsedLibraries todo -func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !sketchError && !l.verbose { - return - } - - res := "" - for header, libResResult := range l.librariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if sketchError { - l.verboseWarnFn(res) - } else { - l.verboseInfoFn(res) - } - // todo why?? should we remove this? - time.Sleep(100 * time.Millisecond) -} - -// AppendIncludeFolder todo should rename this, probably after refactoring the -// container_find_includes command. -//func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { -// ctx.IncludeFolders = append(ctx.IncludeFolders, folder) -// cache.ExpectEntry(sourceFilePath, include, folder) -//} - -// LibrariesLoader todo -func LibrariesLoader( - useCachedLibrariesResolution bool, - librariesManager *librariesmanager.LibrariesManager, - builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, - actualPlatform, targetPlatform *cores.PlatformRelease, -) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { - verboseOut := &bytes.Buffer{} - lm := librariesManager - if useCachedLibrariesResolution { - // Since we are using the cached libraries resolution - // the library manager is not needed. - lm = librariesmanager.NewLibraryManager(nil, nil) - } - if librariesManager == nil { - lm = librariesmanager.NewLibraryManager(nil, nil) - - builtInLibrariesFolders := builtInLibrariesDirs - if builtInLibrariesFolders != nil { - if err := builtInLibrariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) - } - - if actualPlatform != targetPlatform { - lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) - } - lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) - - librariesFolders := otherLibrariesDirs - if err := librariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - for _, folder := range librariesFolders { - lm.AddLibrariesDir(folder, libraries.User) - } - - for _, status := range lm.RescanLibraries() { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. - verboseOut.Write([]byte(status.Message())) - } - - for _, dir := range libraryDirs { - // Libraries specified this way have top priority - if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - } - - resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if actualPlatform != targetPlatform { - if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - return lm, resolver, verboseOut.Bytes(), nil -} diff --git a/arduino/builder/preprocessor/arduino_preprocessor.go b/arduino/builder/preprocessor/arduino_preprocessor.go index 8629ec211ba..a81d4bff612 100644 --- a/arduino/builder/preprocessor/arduino_preprocessor.go +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -21,9 +21,9 @@ import ( "path/filepath" "runtime" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 476c1f2f5b4..9f63eaa93df 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go new file mode 100644 index 00000000000..64c1a54dbdd --- /dev/null +++ b/arduino/builder/utils/utils.go @@ -0,0 +1,185 @@ +package utils + +import ( + "os" + "strings" + "unicode" + + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { + logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) + if objectFile == nil || dependencyFile == nil { + logrus.Debugf("Not found: nil") + return false, nil + } + + sourceFile = sourceFile.Clean() + sourceFileStat, err := sourceFile.Stat() + if err != nil { + return false, errors.WithStack(err) + } + + objectFile = objectFile.Clean() + objectFileStat, err := objectFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", objectFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + dependencyFile = dependencyFile.Clean() + dependencyFileStat, err := dependencyFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", dependencyFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, objectFile) + return false, nil + } + if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) + return false, nil + } + + rows, err := dependencyFile.ReadFileAsLines() + if err != nil { + return false, errors.WithStack(err) + } + + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) + + if len(rows) == 0 { + return true, nil + } + + firstRow := rows[0] + if !strings.HasSuffix(firstRow, ":") { + logrus.Debugf("No colon in first line of depfile") + return false, nil + } + objFileInDepFile := firstRow[:len(firstRow)-1] + if objFileInDepFile != objectFile.String() { + logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) + return false, nil + } + + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + return false, nil + } + + rows = rows[1:] + for _, row := range rows { + depStat, err := os.Stat(row) + if err != nil && !os.IsNotExist(err) { + // There is probably a parsing error of the dep file + // Ignore the error and trigger a full rebuild anyway + logrus.WithError(err).Debugf("Failed to read: %v", row) + return false, nil + } + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", row) + return false, nil + } + if depStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", row, objectFile) + return false, nil + } + } + + return true, nil +} + +func removeEndingBackSlash(s string) string { + return strings.TrimSuffix(s, "\\") +} + +func unescapeDep(s string) string { + s = strings.ReplaceAll(s, "\\ ", " ") + s = strings.ReplaceAll(s, "\\\t", "\t") + s = strings.ReplaceAll(s, "\\#", "#") + s = strings.ReplaceAll(s, "$$", "$") + s = strings.ReplaceAll(s, "\\\\", "\\") + return s +} + +// Normalizes an UTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + +var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} + +// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func FilterOutSCCS(file *paths.Path) bool { + return !SOURCE_CONTROL_FOLDERS[file.Base()] +} + +// FilterReadableFiles is a ReadDirFilter that accepts only readable files +func FilterReadableFiles(file *paths.Path) bool { + // See if the file is readable by opening it + f, err := file.Open() + if err != nil { + return false + } + f.Close() + return true +} + +// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") + +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { + fileFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + paths.FilterOutDirectories(), + FilterReadableFiles, + ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } + if recurse { + dirFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + ) + return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) + } + return dir.ReadDir(fileFilter) +} + +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 96f30a83f39..1daa6f31714 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -24,6 +24,7 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -254,7 +255,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream libsManager = lm } useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() - libsManager, libsResolver, verboseOut, err := bldr.LibrariesLoader( + libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( useCachedLibrariesResolution, libsManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, @@ -267,12 +268,15 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Warn(string(verboseOut)) } - builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + builderCtx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( libsManager, libsResolver, builderCtx.Verbose, useCachedLibrariesResolution, + req.GetCreateCompilationDatabaseOnly(), func(msg string) { builderCtx.Info(msg) }, func(msg string) { builderCtx.Warn(msg) }, + func(data []byte) { builderCtx.WriteStdout(data) }, + func(data []byte) { builderCtx.WriteStderr(data) }, ) defer func() { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 8a8c8253d26..c9336d24528 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -52,7 +52,7 @@ func (s *Builder) Run(ctx *types.Context) error { }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -144,7 +144,7 @@ func PreprocessSketch(ctx *types.Context) error { preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor } normalOutput, verboseOutput, err := preprocessorImpl( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) if ctx.Verbose { ctx.WriteStdout(verboseOutput) @@ -172,7 +172,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { return _err }), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -220,3 +220,19 @@ func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) } + +func findIncludes(ctx *types.Context) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + return nil + }) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index f7cd0348c55..a827e496dac 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -24,16 +24,15 @@ import ( "strings" "sync" + bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/i18n" - f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) var tr = i18n.Tr @@ -48,7 +47,7 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } targetModTime := targetStat.ModTime() - files, err := utils.FindFilesInFolder(dir, true, extensions...) + files, err := bUtils.FindFilesInFolder(dir, true, extensions...) if err != nil { return false, err } @@ -78,7 +77,7 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build validExtensions = append(validExtensions, ext) } - sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) if err != nil { return nil, err } @@ -169,7 +168,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { return nil, errors.WithStack(err) } @@ -205,120 +204,6 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return objectFile, nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { - logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) - if objectFile == nil || dependencyFile == nil { - logrus.Debugf("Not found: nil") - return false, nil - } - - sourceFile = sourceFile.Clean() - sourceFileStat, err := sourceFile.Stat() - if err != nil { - return false, errors.WithStack(err) - } - - objectFile = objectFile.Clean() - objectFileStat, err := objectFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", objectFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - dependencyFile = dependencyFile.Clean() - dependencyFileStat, err := dependencyFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", dependencyFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, objectFile) - return false, nil - } - if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) - return false, nil - } - - rows, err := dependencyFile.ReadFileAsLines() - if err != nil { - return false, errors.WithStack(err) - } - - rows = f.Map(rows, removeEndingBackSlash) - rows = f.Map(rows, strings.TrimSpace) - rows = f.Map(rows, unescapeDep) - rows = f.Filter(rows, f.NotEquals("")) - - if len(rows) == 0 { - return true, nil - } - - firstRow := rows[0] - if !strings.HasSuffix(firstRow, ":") { - logrus.Debugf("No colon in first line of depfile") - return false, nil - } - objFileInDepFile := firstRow[:len(firstRow)-1] - if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) - return false, nil - } - - // The first line of the depfile contains the path to the object file to generate. - // The second line of the depfile contains the path to the source file. - // All subsequent lines contain the header files necessary to compile the object file. - - // If we don't do this check it might happen that trying to compile a source file - // that has the same name but a different path wouldn't recreate the object file. - if sourceFile.String() != strings.Trim(rows[1], " ") { - return false, nil - } - - rows = rows[1:] - for _, row := range rows { - depStat, err := os.Stat(row) - if err != nil && !os.IsNotExist(err) { - // There is probably a parsing error of the dep file - // Ignore the error and trigger a full rebuild anyway - logrus.WithError(err).Debugf("Failed to read: %v", row) - return false, nil - } - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", row) - return false, nil - } - if depStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", row, objectFile) - return false, nil - } - } - - return true, nil -} - -func unescapeDep(s string) string { - s = strings.Replace(s, "\\ ", " ", -1) - s = strings.Replace(s, "\\\t", "\t", -1) - s = strings.Replace(s, "\\#", "#", -1) - s = strings.Replace(s, "$$", "$", -1) - s = strings.Replace(s, "\\\\", "\\", -1) - return s -} - -func removeEndingBackSlash(s string) string { - return strings.TrimSuffix(s, "\\") -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go deleted file mode 100644 index b90c25d46d5..00000000000 --- a/legacy/builder/container_find_includes.go +++ /dev/null @@ -1,474 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -/* - -Include detection - -This code is responsible for figuring out what libraries the current -sketch needs an populates both Context.ImportedLibraries with a list of -Library objects, and Context.IncludeFolders with a list of folders to -put on the include path. - -Simply put, every #include in a source file pulls in the library that -provides that source file. This includes source files in the selected -libraries, so libraries can recursively include other libraries as well. - -To implement this, the gcc preprocessor is used. A queue is created -containing, at first, the source files in the sketch. Each of the files -in the queue is processed in turn by running the preprocessor on it. If -the preprocessor provides an error, the output is examined to see if the -error is a missing header file originating from a #include directive. - -The filename is extracted from that #include directive, and a library is -found that provides it. If multiple libraries provide the same file, one -is slected (how this selection works is not described here, see the -ResolveLibrary function for that). The library selected in this way is -added to the include path through Context.IncludeFolders and the list of -libraries to include in the link through Context.ImportedLibraries. - -Furthermore, all of the library source files are added to the queue, to -be processed as well. When the preprocessor completes without showing an -#include error, processing of the file is complete and it advances to -the next. When no library can be found for a included filename, an error -is shown and the process is aborted. - -Caching - -Since this process is fairly slow (requiring at least one invocation of -the preprocessor per source file), its results are cached. - -Just caching the complete result (i.e. the resulting list of imported -libraries) seems obvious, but such a cache is hard to invalidate. Making -a list of all the source and header files used to create the list and -check if any of them changed is probably feasible, but this would also -require caching the full list of libraries to invalidate the cache when -the include to library resolution might have a different result. Another -downside of a complete cache is that any changes requires re-running -everything, even if no includes were actually changed. - -Instead, caching happens by keeping a sort of "journal" of the steps in -the include detection, essentially tracing each file processed and each -include path entry added. The cache is used by retracing these steps: -The include detection process is executed normally, except that instead -of running the preprocessor, the include filenames are (when possible) -read from the cache. Then, the include file to library resolution is -again executed normally. The results are checked against the cache and -as long as the results match, the cache is considered valid. - -When a source file (or any of the files it includes, as indicated by the -.d file) is changed, the preprocessor is executed as normal for the -file, ignoring any includes from the cache. This does not, however, -invalidate the cache: If the results from the preprocessor match the -entries in the cache, the cache remains valid and can again be used for -the next (unchanged) file. - -The cache file uses the JSON format and contains a list of entries. Each -entry represents a discovered library and contains: - - Sourcefile: The source file that the include was found in - - Include: The included filename found - - Includepath: The addition to the include path - -There are also some special entries: - - When adding the initial include path entries, such as for the core - and variant paths. These are not discovered, so the Sourcefile and - Include fields will be empty. - - When a file contains no (more) missing includes, an entry with an - empty Include and IncludePath is generated. - -*/ - -package builder - -import ( - "encoding/json" - "fmt" - "os/exec" - "time" - - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ContainerFindIncludes struct{} - -func (s *ContainerFindIncludes) Run(ctx *types.Context) error { - err := s.findIncludes(ctx) - if err != nil && ctx.OnlyUpdateCompilationDatabase { - ctx.Info( - fmt.Sprintf("%s: %s", - tr("An error occurred detecting libraries"), - tr("the compilation database may be incomplete or inaccurate"))) - return nil - } - return err -} - -func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { - librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { - return err - } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { - return err - } - if ctx.Verbose { - ctx.Info("Using cached library discovery: " + librariesResolutionCache.String()) - } - return nil - } - - cachePath := ctx.BuildPath.Join("includes.cache") - cache := readCache(cachePath) - - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.core.path")) - if ctx.BuildProperties.Get("build.variant.path") != "" { - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) - } - - sourceFileQueue := &types.UniqueSourceFileQueue{} - - if !ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() { - sketch := ctx.Sketch - mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(mergedfile) - - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */) - srcSubfolderPath := ctx.SketchBuildPath.Join("src") - if srcSubfolderPath.IsDir() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) - } - - for !sourceFileQueue.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFileQueue) - if err != nil { - cachePath.Remove() - return errors.WithStack(err) - } - } - - // Finalize the cache - cache.ExpectEnd() - if err := writeCache(cache, cachePath); err != nil { - return errors.WithStack(err) - } - } - - if err := failIfImportedLibraryIsWrong(ctx); err != nil { - return errors.WithStack(err) - } - - if d, err := json.Marshal(ctx.IncludeFolders); err != nil { - return err - } else if err := librariesResolutionCache.WriteFile(d); err != nil { - return err - } - - return nil -} - -// Append the given folder to the include path and match or append it to -// the cache. sourceFilePath and include indicate the source of this -// include (e.g. what #include line in what file it was resolved from) -// and should be the empty string for the default include folders, like -// the core or variant. -func appendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { - ctx.IncludeFolders = append(ctx.IncludeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) -} - -type includeCacheEntry struct { - Sourcefile *paths.Path - Include string - Includepath *paths.Path -} - -func (entry *includeCacheEntry) String() string { - return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", - entry.Sourcefile, entry.Include, entry.Includepath) -} - -func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { - return entry.String() == other.String() -} - -type includeCache struct { - // Are the cache contents valid so far? - valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// Check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// Check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Read the cache from the given file -func readCache(path *paths.Path) *includeCache { - bytes, err := path.ReadFile() - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result.valid = true - return result -} - -// Write the given cache to the given file if it is invalidated. If the -// cache is still valid, just update the timestamps of the file. -func writeCache(cache *includeCache, path *paths.Path) error { - // If the cache was still valid all the way, just touch its file - // (in case any source file changed without influencing the - // includes). If it was invalidated, overwrite the cache with - // the new contents. - if cache.valid { - path.Chtimes(time.Now(), time.Now()) - } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") - if err != nil { - return errors.WithStack(err) - } - err = path.WriteFile(bytes) - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - -func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error { - sourceFile := sourceFileQueue.Pop() - sourcePath := sourceFile.SourcePath() - targetFilePath := paths.NullPath() - depPath := sourceFile.DepfilePath() - objPath := sourceFile.ObjectPath() - - // TODO: This should perhaps also compare against the - // include.cache file timestamp. Now, it only checks if the file - // changed after the object file was generated, but if it - // changed between generating the cache and the object file, - // this could show the file as unchanged when it really is - // changed. Changing files during a build isn't really - // supported, but any problems from it should at least be - // resolved when doing another build, which is not currently the - // case. - // TODO: This reads the dependency file, but the actual building - // does it again. Should the result be somehow cached? Perhaps - // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) - if err != nil { - return errors.WithStack(err) - } - - first := true - for { - cache.ExpectFile(sourcePath) - - // Libraries may require the "utility" directory to be added to the include - // search path, but only for the source code of the library, so we temporary - // copy the current search path list and add the library' utility directory - // if needed. - includeFolders := ctx.IncludeFolders - if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { - includeFolders = append(includeFolders, extraInclude) - } - - var preprocErr error - var preprocStderr []byte - - var missingIncludeH string - if unchanged && cache.valid { - missingIncludeH = cache.Next().Include - if first && ctx.Verbose { - ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) - } - } else { - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - // Unwrap error and see if it is an ExitError. - if preprocErr == nil { - // Preprocessor successful, done - missingIncludeH = "" - } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { - // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors - return errors.WithStack(preprocErr) - } else { - missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) - if missingIncludeH == "" && ctx.Verbose { - ctx.Info(tr("Error while detecting libraries included by %[1]s", sourcePath)) - } - } - } - - if missingIncludeH == "" { - // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) - return nil - } - - library := ResolveLibrary(ctx, missingIncludeH) - if library == nil { - // Library could not be resolved, show error - if preprocErr == nil || preprocStderr == nil { - // Filename came from cache, so run preprocessor to obtain error to show - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - if preprocErr == nil { - // If there is a missing #include in the cache, but running - // gcc does not reproduce that, there is something wrong. - // Returning an error here will cause the cache to be - // deleted, so hopefully the next compilation will succeed. - return errors.New(tr("Internal error in cache")) - } - } - ctx.WriteStderr(preprocStderr) - return errors.WithStack(preprocErr) - } - - // Add this library to the list of libraries, the - // include path and queue its source files for further - // include scanning - ctx.SketchLibrariesDetector.AppendImportedLibraries(library) - appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) - - if library.Precompiled && library.PrecompiledWithSources { - // Fully precompiled libraries should have no dependencies to avoid ABI breakage - if ctx.Verbose { - ctx.Info(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) - } - } else { - for _, sourceDir := range library.SourceDirs() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) - } - } - first = false - } -} - -func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { - sourceFileExtensions := []string{} - for k := range globals.SourceFilesValidExtensions { - sourceFileExtensions = append(sourceFileExtensions, k) - } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) - if err != nil { - return errors.WithStack(err) - } - - for _, filePath := range filePaths { - sourceFile, err := types.MakeSourceFile(ctx, origin, filePath) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(sourceFile) - } - - return nil -} - -func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - return ctx.SketchLibrariesDetector.ResolveLibrary(header, ctx.TargetPlatform.Platform.Architecture) -} - -func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { - return nil - } - - for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { - if !library.IsLegacy { - if library.InstallDir.Join("arch").IsDir() { - return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) - } - for _, propName := range libraries.MandatoryProperties { - if !library.Properties.ContainsKey(propName) { - return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) - } - } - if library.Layout == libraries.RecursiveLayout { - if library.UtilityDir != nil { - return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) - } - } - } - } - - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 37b60e92fac..45f68ef9ed4 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -26,11 +26,11 @@ import ( properties "github.com/arduino/go-properties-orderedmap" "golang.org/x/exp/slices" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" ) type ExportProjectCMake struct { diff --git a/legacy/builder/includes_finder_with_regexp.go b/legacy/builder/includes_finder_with_regexp.go deleted file mode 100644 index 03017c67048..00000000000 --- a/legacy/builder/includes_finder_with_regexp.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "regexp" - "strings" -) - -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") - -func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) - if match != nil { - return strings.TrimSpace(match[1]) - } - return findIncludeForOldCompilers(source) -} - -func findIncludeForOldCompilers(source string) string { - lines := strings.Split(source, "\n") - for _, line := range lines { - splittedLine := strings.Split(line, ":") - for i := range splittedLine { - if strings.Contains(splittedLine[i], "fatal error") { - return strings.TrimSpace(splittedLine[i+1]) - } - } - } - return "" -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 5c94573f349..455588f7444 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -28,7 +28,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 3d28fbf5520..88106c9a9de 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,12 +18,12 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -37,7 +37,8 @@ type LibrariesBuilder struct{} func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 300cb6836af..398ba87de8b 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,10 +16,10 @@ package phases import ( + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" ) @@ -28,7 +28,8 @@ type SketchBuilder struct{} func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index c4fc3a051b6..10b4f4a4a24 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -138,8 +138,11 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat lm, libsResolver, ctx.Verbose, false, + false, func(msg string) { ctx.Info(msg) }, func(msg string) { ctx.Warn(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, ) } diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index 5c04b1cee10..ab49dc9e26f 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -38,7 +38,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -61,7 +61,7 @@ func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -81,7 +81,7 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { ctx := &types.Context{} ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.LibrariesBuildPath = temp diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go deleted file mode 100644 index b4de7a1a18a..00000000000 --- a/legacy/builder/types/accessories.go +++ /dev/null @@ -1,41 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package types - -import "golang.org/x/exp/slices" - -type UniqueSourceFileQueue []*SourceFile - -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { - *queue = append(*queue, value) - } -} - -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { - return slices.ContainsFunc(queue, target.Equals) -} - -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue UniqueSourceFileQueue) Empty() bool { - return len(queue) == 0 -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index a78530387e9..a6570bf3268 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -62,7 +63,7 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { Builder *builder.Builder - SketchLibrariesDetector *builder.SketchLibrariesDetector + SketchLibrariesDetector *detector.SketchLibrariesDetector // Build options HardwareDirs paths.PathList @@ -100,9 +101,6 @@ type Context struct { Sketch *sketch.Sketch WarningsLevel string - // Libraries handling - IncludeFolders paths.PathList - // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index d84864eecfe..8678fd2fe51 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -15,86 +15,6 @@ package types -import ( - "fmt" - - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/sketch" - paths "github.com/arduino/go-paths-helper" -) - -type SourceFile struct { - // Path to the source file within the sketch/library root folder - relativePath *paths.Path - - // ExtraIncludePath contains an extra include path that must be - // used to compile this source file. - // This is mainly used for source files that comes from old-style libraries - // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. - extraIncludePath *paths.Path - - // The source root for the given origin, where its source files - // can be found. Prepending this to SourceFile.RelativePath will give - // the full path to that source file. - sourceRoot *paths.Path - - // The build root for the given origin, where build products will - // be placed. Any directories inside SourceFile.RelativePath will be - // appended here. - buildRoot *paths.Path -} - -func (f *SourceFile) Equals(g *SourceFile) bool { - return f.relativePath.EqualsTo(g.relativePath) && - f.buildRoot.EqualsTo(g.buildRoot) && - f.sourceRoot.EqualsTo(g.sourceRoot) -} - -// Create a SourceFile containing the given source file path within the -// given origin. The given path can be absolute, or relative within the -// origin's root source folder -func MakeSourceFile(ctx *Context, origin interface{}, path *paths.Path) (*SourceFile, error) { - res := &SourceFile{} - - switch o := origin.(type) { - case *sketch.Sketch: - res.buildRoot = ctx.SketchBuildPath - res.sourceRoot = ctx.SketchBuildPath - case *libraries.Library: - res.buildRoot = ctx.LibrariesBuildPath.Join(o.DirName) - res.sourceRoot = o.SourceDir - res.extraIncludePath = o.UtilityDir - default: - panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) - } - - if path.IsAbs() { - var err error - path, err = res.sourceRoot.RelTo(path) - if err != nil { - return nil, err - } - } - res.relativePath = path - return res, nil -} - -func (f *SourceFile) ExtraIncludePath() *paths.Path { - return f.extraIncludePath -} - -func (f *SourceFile) SourcePath() *paths.Path { - return f.sourceRoot.JoinPath(f.relativePath) -} - -func (f *SourceFile) ObjectPath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".o") -} - -func (f *SourceFile) DepfilePath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".d") -} - type Command interface { Run(ctx *Context) error } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 102b007883f..1cddbd3bb48 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -20,42 +20,12 @@ import ( "os" "os/exec" "strings" - "unicode" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/text/runes" - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" ) -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} - -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") - -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] -} - -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { - // See if the file is readable by opening it - f, err := file.Open() - if err != nil { - return false - } - f.Close() - return true -} - -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.Replace(arg, "\\", "\\\\", -1) @@ -126,29 +96,6 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { - fileFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - paths.FilterOutDirectories(), - FilterReadableFiles, - ) - if len(extensions) > 0 { - fileFilter = paths.AndFilter( - paths.FilterSuffixes(extensions...), - fileFilter, - ) - } - if recurse { - dirFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - ) - return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) - } - return dir.ReadDir(fileFilter) -} - type loggerAction struct { onlyIfVerbose bool warn bool @@ -169,11 +116,3 @@ func (l *loggerAction) Run(ctx *types.Context) error { func LogIfVerbose(warn bool, msg string) types.Command { return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} } - -// Normalizes an UTF8 byte slice -// TODO: use it more often troughout all the project (maybe on logger interface?) -func NormalizeUTF8(buf []byte) []byte { - t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) - result, _, _ := transform.Bytes(t, buf) - return result -}