From 85fa332d4bb4a0c4d6763e54681feaa92594491c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 31 Aug 2023 19:21:41 +0200 Subject: [PATCH 01/18] Refactor legacy LibrariesLoader command --- legacy/builder/container_setup.go | 29 +++---- legacy/builder/libraries_loader.go | 71 +++++++++-------- legacy/builder/test/libraries_loader_test.go | 82 ++++++++++---------- 3 files changed, 92 insertions(+), 90 deletions(-) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 8a99df5c8a9..4d2d8a6d22d 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -33,20 +33,23 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) ctx.CoreBuildPath = coreBuildPath ctx.WarningsLevel = warningsLevel - commands := []types.Command{ - &FailIfBuildPathEqualsSketchPath{}, - &LibrariesLoader{}, + if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { + return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } - ctx.Progress.AddSubSteps(len(commands)) - defer ctx.Progress.RemoveSubSteps() - for _, command := range commands { - PrintRingNameIfDebug(ctx, command) - err := command.Run(ctx) - if err != nil { - return errors.WithStack(err) - } - ctx.Progress.CompleteStep() - ctx.PushProgress() + + lm, libsResolver, verboseOut, err := LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + if err != nil { + return errors.WithStack(err) + } + + ctx.LibrariesManager = lm + ctx.LibrariesResolver = libsResolver + if ctx.Verbose { + ctx.Warn(string(verboseOut)) } return nil } diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go index b41c49edcb1..fcdf7ed89f9 100644 --- a/legacy/builder/libraries_loader.go +++ b/legacy/builder/libraries_loader.go @@ -16,41 +16,48 @@ package builder import ( + "bytes" + + "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/go-paths-helper" "github.com/pkg/errors" ) -type LibrariesLoader struct{} - -func (s *LibrariesLoader) Run(ctx *types.Context) error { - if ctx.UseCachedLibrariesResolution { +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { // Since we are using the cached libraries resolution // the library manager is not needed. - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm - } else if ctx.LibrariesManager == nil { - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) - builtInLibrariesFolders := ctx.BuiltInLibrariesDirs + builtInLibrariesFolders := builtInLibrariesDirs if builtInLibrariesFolders != nil { if err := builtInLibrariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) } - if ctx.ActualPlatform != ctx.TargetPlatform { - lm.AddPlatformReleaseLibrariesDir(ctx.ActualPlatform, libraries.ReferencedPlatformBuiltIn) + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) } - lm.AddPlatformReleaseLibrariesDir(ctx.TargetPlatform, libraries.PlatformBuiltIn) + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) - librariesFolders := ctx.OtherLibrariesDirs + librariesFolders := otherLibrariesDirs if err := librariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } for _, folder := range librariesFolders { lm.AddLibrariesDir(folder, libraries.User) @@ -63,35 +70,31 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { // I have no intention right now to start a refactoring of the legacy package too, so // here's this shitty solution for now. // When we're gonna refactor the legacy package this will be gone. - if ctx.Verbose { - ctx.Warn(status.Message()) - } + verboseOut.Write([]byte(status.Message())) } - for _, dir := range ctx.LibraryDirs { + for _, dir := range libraryDirs { // Libraries specified this way have top priority if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return err + return nil, nil, nil, errors.WithStack(err) } } } resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if err := resolver.ScanUserAndUnmanagedLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.TargetPlatform); err != nil { - return errors.WithStack(err) + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if ctx.ActualPlatform != ctx.TargetPlatform { - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.ActualPlatform); err != nil { - return errors.WithStack(err) + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) } } - ctx.LibrariesResolver = resolver - - return nil + return lm, resolver, verboseOut.Bytes(), nil } diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index a18880fed92..7b7fcb12750 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -28,9 +28,9 @@ import ( "github.com/stretchr/testify/require" ) -func extractLibraries(ctx *types.Context) []*libraries.Library { +func extractLibraries(libs map[string]libraries.List) []*libraries.Library { res := []*libraries.Library{} - for _, lib := range ctx.LibrariesManager.Libraries { + for _, lib := range libs { for _, libAlternative := range lib { res = append(res, libAlternative) } @@ -47,21 +47,20 @@ func TestLoadLibrariesAVR(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libs := extractLibraries(ctx) + libs := extractLibraries(lm.Libraries) require.Equal(t, 24, len(libs)) sort.Sort(ByLibraryName(libs)) @@ -124,21 +123,21 @@ func TestLoadLibrariesAVR(t *testing.T) { idx++ require.Equal(t, "Wire", libs[idx].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs = libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Adafruit_PN532.h") + libs = libsResolver.AlternativesFor("Adafruit_PN532.h") require.Len(t, libs, 1) require.Equal(t, "Adafruit PN532", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -152,21 +151,20 @@ func TestLoadLibrariesSAM(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "sam", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libraries := extractLibraries(ctx) + libraries := extractLibraries(lm.Libraries) require.Equal(t, 22, len(libraries)) sort.Sort(ByLibraryName(libraries)) @@ -206,17 +204,17 @@ func TestLoadLibrariesSAM(t *testing.T) { idx++ require.Equal(t, "Wire", libraries[idx].Name) - libs := ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs := libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -230,15 +228,14 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) @@ -254,15 +251,14 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 4, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) From 82f9dd77eae44d648a73ae6f7c172b23b38451fe Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 31 Aug 2023 19:34:28 +0200 Subject: [PATCH 02/18] move LibrariesLoader under arduino/builder --- .../libraries_loader.go => arduino/builder/libraries.go | 0 legacy/builder/container_setup.go | 3 ++- legacy/builder/test/libraries_loader_test.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename legacy/builder/libraries_loader.go => arduino/builder/libraries.go (100%) diff --git a/legacy/builder/libraries_loader.go b/arduino/builder/libraries.go similarity index 100% rename from legacy/builder/libraries_loader.go rename to arduino/builder/libraries.go diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 4d2d8a6d22d..a0ee98d86f6 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -16,6 +16,7 @@ package builder import ( + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" ) @@ -37,7 +38,7 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } - lm, libsResolver, verboseOut, err := LibrariesLoader( + lm, libsResolver, verboseOut, err := builder.LibrariesLoader( ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 7b7fcb12750..8ef2c629a24 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -21,7 +21,7 @@ import ( "testing" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" From 11e08ceecdc239ddd2cfe9d807109fdc1abd241f Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 15:50:04 +0200 Subject: [PATCH 03/18] remove usless nil check --- .../add_additional_entries_to_context.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index 843cbdcd79e..0f3a2fb3f57 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -22,22 +22,17 @@ import ( ) func AddAdditionalEntriesToContext(buildPath *paths.Path, warningLevel string) (*paths.Path, *paths.Path, *paths.Path, string, error) { - var sketchBuildPath, librariesBuildPath, coreBuildPath *paths.Path - var err error - - if buildPath != nil { - sketchBuildPath, err = buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - librariesBuildPath, err = buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - coreBuildPath, err = buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) + } + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) + } + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) } if warningLevel == "" { From 2bd1024bcb13935ea4998e6180fe30cce92e08b2 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:02:32 +0200 Subject: [PATCH 04/18] remove AddAdditionalEntries func, in favour of initializing them in the compile command --- commands/compile/compile.go | 25 +++++++++ .../add_additional_entries_to_context.go | 43 --------------- legacy/builder/container_setup.go | 10 ---- .../add_additional_entries_to_context_test.go | 55 ------------------- legacy/builder/test/builder_test.go | 12 ++++ 5 files changed, 37 insertions(+), 108 deletions(-) delete mode 100644 legacy/builder/add_additional_entries_to_context.go delete mode 100644 legacy/builder/test/add_additional_entries_to_context_test.go diff --git a/commands/compile/compile.go b/commands/compile/compile.go index b374213f2d3..4a652b8dd6e 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -18,6 +18,7 @@ package compile import ( "context" "fmt" + "go/build" "io" "sort" "strings" @@ -33,6 +34,7 @@ import ( "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/internal/inventory" "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -189,6 +191,25 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.BuildPath = buildPath builderCtx.ProgressCB = progressCB + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return nil, err + } + builderCtx.SketchBuildPath = sketchBuildPath + + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return nil, err + } + builderCtx.LibrariesBuildPath = librariesBuildPath + + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return nil, err + } + builderCtx.CoreBuildPath = coreBuildPath + //ctx.WarningsLevel = warningsLevel + // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) @@ -202,7 +223,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Verbose = req.GetVerbose() builderCtx.Jobs = int(req.GetJobs()) + builderCtx.WarningsLevel = req.GetWarnings() + if builderCtx.WarningsLevel == "" { + builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL + } if req.GetBuildCachePath() == "" { builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go deleted file mode 100644 index 0f3a2fb3f57..00000000000 --- a/legacy/builder/add_additional_entries_to_context.go +++ /dev/null @@ -1,43 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -func AddAdditionalEntriesToContext(buildPath *paths.Path, warningLevel string) (*paths.Path, *paths.Path, *paths.Path, string, error) { - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - - if warningLevel == "" { - warningLevel = DEFAULT_WARNINGS_LEVEL - } - - return sketchBuildPath, librariesBuildPath, coreBuildPath, warningLevel, nil -} diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index a0ee98d86f6..0a3448352e3 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -24,16 +24,6 @@ import ( type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - if err != nil { - return errors.WithStack(err) - } - ctx.SketchBuildPath = sketchBuildPath - ctx.LibrariesBuildPath = librariesBuildPath - ctx.CoreBuildPath = coreBuildPath - ctx.WarningsLevel = warningsLevel - if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go deleted file mode 100644 index dc15e16697c..00000000000 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "testing" - - "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" - "github.com/stretchr/testify/require" -) - -func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { - ctx := &types.Context{} - - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - NoError(t, err) - - require.Empty(t, sketchBuildPath) - require.Empty(t, librariesBuildPath) - require.Empty(t, coreBuildPath) - - require.NotNil(t, warningsLevel) -} - -func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { - ctx := &types.Context{} - ctx.BuildPath = paths.New("folder") - - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - NoError(t, err) - - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), sketchBuildPath) - require.Equal(t, Abs(t, paths.New("folder", "libraries")), librariesBuildPath) - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), coreBuildPath) - - require.NotNil(t, warningsLevel) -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 322a8fd02be..3e918a785af 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -59,6 +59,18 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.BuildPath = buildPath } + buildPath := ctx.BuildPath + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + NoError(t, err) + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + NoError(t, err) + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + NoError(t, err) + + ctx.SketchBuildPath = sketchBuildPath + ctx.LibrariesBuildPath = librariesBuildPath + ctx.CoreBuildPath = coreBuildPath + // Create a Package Manager from the given context // This should happen only on legacy arduino-builder. // Hopefully this piece will be removed once the legacy package will be cleanedup. From efa7cb7500ba74fd115b4fa079bbcc771f2b6544 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:06:52 +0200 Subject: [PATCH 05/18] move a check directly in the compile command --- commands/compile/compile.go | 7 +++++-- legacy/builder/container_setup.go | 4 ---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 4a652b8dd6e..b860b1bd9dc 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -17,8 +17,8 @@ package compile import ( "context" + "errors" "fmt" - "go/build" "io" "sort" "strings" @@ -208,7 +208,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return nil, err } builderCtx.CoreBuildPath = coreBuildPath - //ctx.WarningsLevel = warningsLevel + + if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { + return nil, errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) + } // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 0a3448352e3..03c9cce88d4 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -24,10 +24,6 @@ import ( type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { - return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - lm, libsResolver, verboseOut, err := builder.LibrariesLoader( ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, From acac706043c39e6ba5319b1f687e80f3bb8ddf16 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:23:54 +0200 Subject: [PATCH 06/18] create the SketchLibrariesDetector struct --- arduino/builder/libraries.go | 92 +++++++++++++++++++++++++++++++++ legacy/builder/types/context.go | 3 +- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index fcdf7ed89f9..ffee5539ec4 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -17,6 +17,7 @@ package builder import ( "bytes" + "fmt" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" @@ -26,6 +27,97 @@ import ( "github.com/pkg/errors" ) +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + importedLibraries libraries.List, + verbose, useCachedLibrariesResolution bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), +) *SketchLibrariesDetector { + + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + + importedLibraries: importedLibraries, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, librariesManager *librariesmanager.LibrariesManager, diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0c887ee6125..9e022e315e9 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -64,7 +64,8 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { - Builder *builder.Builder + Builder *builder.Builder + SketchLibrariesDetector *builder.SketchLibrariesDetector // Build options HardwareDirs paths.PathList From 7edeb03b8017f630ed49a6618faead4f46e8793e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:56:00 +0200 Subject: [PATCH 07/18] move all the logic of ContainerSetupHardwareToolsLibsSketchAndProps in the compile command --- commands/compile/compile.go | 86 +++++++++++++++++++------------ legacy/builder/builder.go | 11 ---- legacy/builder/container_setup.go | 16 ------ 3 files changed, 53 insertions(+), 60 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index b860b1bd9dc..2ca455946c0 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -17,7 +17,6 @@ package compile import ( "context" - "errors" "fmt" "io" "sort" @@ -175,9 +174,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx := &types.Context{} builderCtx.Builder = sketchBuilder builderCtx.PackageManager = pme - if pme.GetProfile() != nil { - builderCtx.LibrariesManager = lm - } builderCtx.TargetBoard = targetBoard builderCtx.TargetPlatform = targetPlatform builderCtx.TargetPackage = targetPackage @@ -185,34 +181,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.RequiredTools = requiredTools builderCtx.BuildProperties = buildProperties builderCtx.CustomBuildProperties = customBuildPropertiesArgs - builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() builderCtx.FQBN = fqbn builderCtx.Sketch = sk builderCtx.BuildPath = buildPath builderCtx.ProgressCB = progressCB - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, err - } - builderCtx.SketchBuildPath = sketchBuildPath - - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, err - } - builderCtx.LibrariesBuildPath = librariesBuildPath - - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, err - } - builderCtx.CoreBuildPath = coreBuildPath - - if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { - return nil, errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) @@ -253,6 +226,57 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly() builderCtx.SourceOverride = req.GetSourceOverride() + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + builderCtx.SketchBuildPath = sketchBuildPath + builderCtx.LibrariesBuildPath = librariesBuildPath + builderCtx.CoreBuildPath = coreBuildPath + + if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { + return r, &arduino.CompileFailedError{ + Message: tr("Sketch cannot be located in build path. Please specify a different build path"), + } + } + + // TODO replace all UseCache call with our SketchLibrariesDetector + builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() + if pme.GetProfile() != nil { + builderCtx.LibrariesManager = lm + } + lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( + builderCtx.UseCachedLibrariesResolution, builderCtx.LibrariesManager, + builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, + builderCtx.ActualPlatform, builderCtx.TargetPlatform, + ) + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + + builderCtx.LibrariesManager = lm + builderCtx.LibrariesResolver = libsResolver + if builderCtx.Verbose { + builderCtx.Warn(string(verboseOut)) + } + + builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + lm, libsResolver, + builderCtx.ImportedLibraries, + builderCtx.Verbose, + builderCtx.UseCachedLibrariesResolution, + func(msg string) { builderCtx.Info(msg) }, + func(msg string) { builderCtx.Warn(msg) }, + ) + defer func() { if p := builderCtx.BuildPath; p != nil { r.BuildPath = p.String() @@ -274,13 +298,9 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } }() + // Just get build properties and exit if req.GetShowProperties() { - // Just get build properties and exit - compileErr := builder.RunParseHardware(builderCtx) - if compileErr != nil { - compileErr = &arduino.CompileFailedError{Message: compileErr.Error()} - } - return r, compileErr + return r, nil } if req.GetPreprocess() { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index bce8bd24d6b..f914e11140f 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -42,8 +42,6 @@ func (s *Builder) Run(ctx *types.Context) error { var _err, mainErr error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, @@ -162,8 +160,6 @@ func (s *Preprocess) Run(ctx *types.Context) error { var _err error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, @@ -217,13 +213,6 @@ func RunBuilder(ctx *types.Context) error { return runCommands(ctx, []types.Command{&Builder{}}) } -func RunParseHardware(ctx *types.Context) error { - commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - return runCommands(ctx, commands) -} - func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 03c9cce88d4..3179ef6c1b2 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -16,27 +16,11 @@ package builder import ( - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" ) type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - lm, libsResolver, verboseOut, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, - ctx.ActualPlatform, ctx.TargetPlatform, - ) - if err != nil { - return errors.WithStack(err) - } - - ctx.LibrariesManager = lm - ctx.LibrariesResolver = libsResolver - if ctx.Verbose { - ctx.Warn(string(verboseOut)) - } return nil } From a086232353da8e39d0d3311c17c1db55efbd40b5 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 18:02:27 +0200 Subject: [PATCH 08/18] remove container_setup and adjust relative tests --- legacy/builder/container_setup.go | 26 ---- legacy/builder/test/builder_test.go | 35 +++++- legacy/builder/test/ctags_runner_test.go | 11 +- legacy/builder/test/hardware_loader_test.go | 9 +- .../test/includes_to_include_folders_test.go | 27 ++-- .../test/merge_sketch_with_bootloader_test.go | 12 -- legacy/builder/test/prototypes_adder_test.go | 117 ++++++------------ .../test/setup_build_properties_test.go | 10 -- legacy/builder/test/tools_loader_test.go | 6 +- 9 files changed, 94 insertions(+), 159 deletions(-) delete mode 100644 legacy/builder/container_setup.go diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go deleted file mode 100644 index 3179ef6c1b2..00000000000 --- a/legacy/builder/container_setup.go +++ /dev/null @@ -1,26 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type ContainerSetupHardwareToolsLibsSketchAndProps struct{} - -func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - return nil -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 3e918a785af..068f24ac3e4 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -41,9 +41,18 @@ func cleanUpBuilderTestContext(t *testing.T, ctx *types.Context) { } } -func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string) *types.Context { +type skipContextPreparationStepName string + +const skipLibraries = skipContextPreparationStepName("libraries") + +func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string, skips ...skipContextPreparationStepName) *types.Context { DownloadCoresAndToolsAndLibraries(t) + stepToSkip := map[skipContextPreparationStepName]bool{} + for _, skip := range skips { + stepToSkip[skip] = true + } + if ctx == nil { ctx = &types.Context{} } @@ -113,6 +122,30 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.RequiredTools = requiredTools } + if ctx.Sketch != nil { + require.False(t, ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical())) + } + + if !stepToSkip[skipLibraries] { + lm, libsResolver, _, err := bldr.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) + + ctx.LibrariesManager = lm + ctx.LibrariesResolver = libsResolver + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + lm, libsResolver, + ctx.ImportedLibraries, + ctx.Verbose, + ctx.UseCachedLibrariesResolution, + func(msg string) { ctx.Info(msg) }, + func(msg string) { ctx.Warn(msg) }, + ) + } + return ctx } diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index dc7fd0de09e..2e39b692320 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -20,20 +20,17 @@ import ( "testing" "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) func ctagsRunnerTestTemplate(t *testing.T, sketchLocation *paths.Path) []byte { - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) - NoError(t, err) - - _, err = ctx.Builder.PrepareSketchBuildPath(nil, ctx.SketchBuildPath) + _, err := ctx.Builder.PrepareSketchBuildPath(nil, ctx.SketchBuildPath) NoError(t, err) source := loadPreprocessedSketch(t, ctx) diff --git a/legacy/builder/test/hardware_loader_test.go b/legacy/builder/test/hardware_loader_test.go index c7e4b0ba387..ae87e1c41c7 100644 --- a/legacy/builder/test/hardware_loader_test.go +++ b/legacy/builder/test/hardware_loader_test.go @@ -33,7 +33,8 @@ func TestLoadHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware")), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -65,7 +66,7 @@ func TestLoadHardwareMixingUserHardwareFolder(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -125,7 +126,7 @@ func TestLoadHardwareWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -166,7 +167,7 @@ func TestLoadLotsOfHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff", "downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index 75cc7517f33..127e953e2c0 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -27,13 +27,12 @@ import ( ) func TestIncludesToIncludeFolders(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -51,13 +50,12 @@ func TestIncludesToIncludeFolders(t *testing.T) { } func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("SketchWithIfDef", "SketchWithIfDef.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("SketchWithIfDef", "SketchWithIfDef.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -74,13 +72,12 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { } func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("sketch9", "sketch9.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch9", "sketch9.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -100,13 +97,12 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { } func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("sketch10", "sketch10.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch10", "sketch10.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -137,7 +133,6 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -169,7 +164,6 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -201,7 +195,6 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -221,13 +214,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { } func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("sketch_with_subfolders", "sketch_with_subfolders.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_with_subfolders", "sketch_with_subfolders.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err diff --git a/legacy/builder/test/merge_sketch_with_bootloader_test.go b/legacy/builder/test/merge_sketch_with_bootloader_test.go index b36102f6c40..43387eeaa8d 100644 --- a/legacy/builder/test/merge_sketch_with_bootloader_test.go +++ b/legacy/builder/test/merge_sketch_with_bootloader_test.go @@ -70,7 +70,6 @@ func TestMergeSketchWithBootloader(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -129,7 +128,6 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -152,15 +150,6 @@ func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) buildPath := ctx.BuildPath - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) @@ -222,7 +211,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 5664b72e7ad..56ddc806c0c 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -37,15 +37,12 @@ func loadPreprocessedSketch(t *testing.T, ctx *types.Context) string { func TestPrototypesAdderBridgeExample(t *testing.T) { sketchLocation := paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -64,14 +61,12 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { } func TestPrototypesAdderSketchWithIfDef(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("SketchWithIfDef", "SketchWithIfDef.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("SketchWithIfDef", "SketchWithIfDef.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -90,14 +85,12 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { } func TestPrototypesAdderBaladuino(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("Baladuino", "Baladuino.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("Baladuino", "Baladuino.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -116,14 +109,12 @@ func TestPrototypesAdderBaladuino(t *testing.T) { } func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -142,14 +133,12 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { } func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.ino"), "arduino:sam:arduino_due_x_dbg") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.ino"), "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -168,14 +157,12 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { } func TestPrototypesAdderLineContinuations(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("LineContinuations", "LineContinuations.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("LineContinuations", "LineContinuations.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -194,14 +181,12 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { } func TestPrototypesAdderStringWithComment(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("StringWithComment", "StringWithComment.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("StringWithComment", "StringWithComment.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -220,14 +205,12 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { } func TestPrototypesAdderSketchWithStruct(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("SketchWithStruct", "SketchWithStruct.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("SketchWithStruct", "SketchWithStruct.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -254,14 +237,12 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { sketchLocation := paths.New("sketch_with_config", "sketch_with_config.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -286,14 +267,12 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { sketchLocation := paths.New("sketch_no_functions_two_files", "sketch_no_functions_two_files.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, paths.New("sketch_no_functions_two_files", "sketch_no_functions_two_files.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_no_functions_two_files", "sketch_no_functions_two_files.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -313,16 +292,14 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { } func TestPrototypesAdderSketchNoFunctions(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("sketch_no_functions", "sketch_no_functions.ino"), "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_no_functions", "sketch_no_functions.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - sketchLocation := paths.New("sketch_no_functions", "sketch_no_functions.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -345,14 +322,12 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { sketchLocation := paths.New("sketch_with_default_args", "sketch_with_default_args.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -374,14 +349,12 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { sketchLocation := paths.New("sketch_with_inline_function", "sketch_with_inline_function.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -414,14 +387,12 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { sketchLocation := paths.New("sketch_with_function_signature_inside_ifdef", "sketch_with_function_signature_inside_ifdef.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -455,7 +426,6 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -488,7 +458,6 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -516,14 +485,12 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { sketchLocation := paths.New("sketch_with_ifdef", "sketch_with_ifdef.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:yun") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:yun") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -548,14 +515,12 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { sketchLocation := paths.New("sketch_with_ifdef", "sketch_with_ifdef.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:sam:arduino_due_x_dbg") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -580,14 +545,12 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { sketchLocation := paths.New("sketch_with_const", "sketch_with_const.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:uno") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -606,14 +569,12 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { } func TestPrototypesAdderSketchWithDosEol(t *testing.T) { - ctx := prepareBuilderTestContext(t, nil, paths.New("eol_processing", "eol_processing.ino"), "arduino:avr:uno") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, paths.New("eol_processing", "eol_processing.ino"), "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err @@ -632,14 +593,12 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { sketchLocation := paths.New("sketch_with_class_and_method_substring", "sketch_with_class_and_method_substring.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:uno") + ctx := &types.Context{Verbose: true} + ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - var _err error commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err diff --git a/legacy/builder/test/setup_build_properties_test.go b/legacy/builder/test/setup_build_properties_test.go index d26a217d398..2c0344812a2 100644 --- a/legacy/builder/test/setup_build_properties_test.go +++ b/legacy/builder/test/setup_build_properties_test.go @@ -19,7 +19,6 @@ import ( "path/filepath" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -118,15 +117,6 @@ func TestSetupBuildPropertiesWithMissingPropsFromParentPlatformTxtFiles(t *testi ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) diff --git a/legacy/builder/test/tools_loader_test.go b/legacy/builder/test/tools_loader_test.go index 7138df09afc..e28f828401d 100644 --- a/legacy/builder/test/tools_loader_test.go +++ b/legacy/builder/test/tools_loader_test.go @@ -66,7 +66,7 @@ func TestLoadTools(t *testing.T) { HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -107,7 +107,7 @@ func TestLoadToolsWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -131,7 +131,7 @@ func TestLoadLotsOfTools(t *testing.T) { HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() From e6eb0552b15ddaf3ef5eab3e1096c9f5386759ca Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:26:01 +0200 Subject: [PATCH 09/18] remove LibraryResolver property from context --- commands/compile/compile.go | 1 - legacy/builder/container_find_includes.go | 47 +---------------------- legacy/builder/test/builder_test.go | 1 - legacy/builder/types/context.go | 1 - 4 files changed, 1 insertion(+), 49 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 2ca455946c0..445f35b7a9b 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -263,7 +263,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } builderCtx.LibrariesManager = lm - builderCtx.LibrariesResolver = libsResolver if builderCtx.Verbose { builderCtx.Warn(string(verboseOut)) } diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 55b60347898..142263fa421 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -444,52 +444,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.Uniqu } func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - resolver := ctx.LibrariesResolver - importedLibraries := ctx.ImportedLibraries - - if ctx.LibrariesResolutionResults == nil { - ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} - } - - candidates := resolver.AlternativesFor(header) - - if ctx.Verbose { - ctx.Info(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - ctx.Info(fmt.Sprintf("ResolveLibrary(%s)", header)) - ctx.Info(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := resolver.ResolveFor(header, ctx.TargetPlatform.Platform.Architecture) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - ctx.LibrariesResolutionResults[header] = types.LibraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected + return ctx.SketchLibrariesDetector.ResolveLibrary(header, ctx.TargetPlatform.Platform.Architecture) } func failIfImportedLibraryIsWrong(ctx *types.Context) error { diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 068f24ac3e4..e8a0bffe745 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -135,7 +135,6 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat NoError(t, err) ctx.LibrariesManager = lm - ctx.LibrariesResolver = libsResolver ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, ctx.ImportedLibraries, diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 9e022e315e9..901ef26b75a 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -105,7 +105,6 @@ type Context struct { // Libraries handling LibrariesManager *librariesmanager.LibrariesManager - LibrariesResolver *librariesresolver.Cpp ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList From 50e8370c332b6a84967dacd91c8c1cabeade08dd Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:31:48 +0200 Subject: [PATCH 10/18] remove UseCachedLibrariesResolution for context --- arduino/builder/libraries.go | 4 ++++ commands/compile/compile.go | 6 +++--- legacy/builder/container_find_includes.go | 4 ++-- legacy/builder/test/builder_test.go | 4 ++-- legacy/builder/test/libraries_loader_test.go | 8 ++++---- legacy/builder/types/context.go | 2 -- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index ffee5539ec4..b620280940e 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -117,6 +117,10 @@ func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { return l.importedLibraries } +func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { + return l.useCachedLibrariesResolution +} + // LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 445f35b7a9b..56ea6728d0a 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -249,12 +249,12 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } // TODO replace all UseCache call with our SketchLibrariesDetector - builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() + useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() if pme.GetProfile() != nil { builderCtx.LibrariesManager = lm } lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( - builderCtx.UseCachedLibrariesResolution, builderCtx.LibrariesManager, + useCachedLibrariesResolution, builderCtx.LibrariesManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) @@ -271,7 +271,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream lm, libsResolver, builderCtx.ImportedLibraries, builderCtx.Verbose, - builderCtx.UseCachedLibrariesResolution, + useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, func(msg string) { builderCtx.Warn(msg) }, ) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 142263fa421..c810a30622a 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -124,7 +124,7 @@ func (s *ContainerFindIncludes) Run(ctx *types.Context) error { func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.UseCachedLibrariesResolution && librariesResolutionCache.Exist() { + if ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() && librariesResolutionCache.Exist() { if d, err := librariesResolutionCache.ReadFile(); err != nil { return err } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { @@ -146,7 +146,7 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { sourceFileQueue := &types.UniqueSourceFileQueue{} - if !ctx.UseCachedLibrariesResolution { + if !ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() { sketch := ctx.Sketch mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index e8a0bffe745..621df16037d 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -128,7 +128,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := bldr.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -139,7 +139,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat lm, libsResolver, ctx.ImportedLibraries, ctx.Verbose, - ctx.UseCachedLibrariesResolution, + false, func(msg string) { ctx.Info(msg) }, func(msg string) { ctx.Warn(msg) }, ) diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 8ef2c629a24..66e27202fa5 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -48,7 +48,7 @@ func TestLoadLibrariesAVR(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -152,7 +152,7 @@ func TestLoadLibrariesSAM(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -229,7 +229,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -252,7 +252,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 901ef26b75a..edf938877bb 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -27,7 +27,6 @@ import ( "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -108,7 +107,6 @@ type Context struct { ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList - UseCachedLibrariesResolution bool // C++ Parsing LineOffset int From cff9cdc2fb27cc2971a96897d6fced33efb680c8 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:56:05 +0200 Subject: [PATCH 11/18] remove ImportedLibraries from context --- arduino/builder/libraries.go | 25 +++++++++++++------ commands/compile/compile.go | 3 +-- legacy/builder/container_find_includes.go | 6 ++--- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/phases/libraries_builder.go | 2 +- .../print_used_libraries_if_verbose.go | 4 +-- legacy/builder/test/builder_test.go | 1 - .../test/includes_to_include_folders_test.go | 16 ++++++------ .../unused_compiled_libraries_remover_test.go | 15 ++++++++--- legacy/builder/types/context.go | 2 -- .../unused_compiled_libraries_remover.go | 2 +- .../warn_about_arch_incompatible_libraries.go | 2 +- 12 files changed, 48 insertions(+), 32 deletions(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index b620280940e..d6f273f5f4c 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -40,21 +40,18 @@ type SketchLibrariesDetector struct { verbose bool verboseInfoFn func(msg string) verboseWarnFn func(msg string) - - importedLibraries libraries.List - librariesResolutionResults map[string]libraryResolutionResult + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult } // NewSketchLibrariesDetector todo func NewSketchLibrariesDetector( lm *librariesmanager.LibrariesManager, libsResolver *librariesresolver.Cpp, - importedLibraries libraries.List, verbose, useCachedLibrariesResolution bool, verboseInfoFn func(msg string), verboseWarnFn func(msg string), ) *SketchLibrariesDetector { - return &SketchLibrariesDetector{ librariesManager: lm, librariesResolver: libsResolver, @@ -63,8 +60,7 @@ func NewSketchLibrariesDetector( verbose: verbose, verboseInfoFn: verboseInfoFn, verboseWarnFn: verboseWarnFn, - - importedLibraries: importedLibraries, + importedLibraries: libraries.List{}, } } @@ -112,15 +108,30 @@ func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *l return selected } +// ImportedLibraries todo func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { // TODO understand if we have to do a deepcopy return l.importedLibraries } +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// UseCachedLibrariesResolution todo func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { return l.useCachedLibrariesResolution } +// AppendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +//func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { +// ctx.IncludeFolders = append(ctx.IncludeFolders, folder) +// cache.ExpectEntry(sourceFilePath, include, folder) +//} + // LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 56ea6728d0a..c5f36dbcc7c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -269,7 +269,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, - builderCtx.ImportedLibraries, builderCtx.Verbose, useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, @@ -313,7 +312,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream defer func() { importedLibs := []*rpc.Library{} - for _, lib := range builderCtx.ImportedLibraries { + for _, lib := range builderCtx.SketchLibrariesDetector.ImportedLibraries() { rpcLib, err := lib.ToRPCLibrary() if err != nil { msg := tr("Error getting information for library %s", lib.Name) + ": " + err.Error() + "\n" diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index c810a30622a..b90c25d46d5 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -405,7 +405,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu // Add this library to the list of libraries, the // include path and queue its source files for further // include scanning - ctx.ImportedLibraries = append(ctx.ImportedLibraries, library) + ctx.SketchLibrariesDetector.AppendImportedLibraries(library) appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) if library.Precompiled && library.PrecompiledWithSources { @@ -448,11 +448,11 @@ func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { } func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.ImportedLibraries) == 0 { + if len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { return nil } - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { if !library.IsLegacy { if library.InstallDir.Join("arch").IsDir() { return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index acbce09e2cb..37b60e92fac 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -198,7 +198,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { cmakeFile := cmakeFolder.Join("CMakeLists.txt") dynamicLibsFromPkgConfig := map[string]bool{} - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { // Copy used libraries in the correct folder libDir := libBaseFolder.Join(library.DirName) mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 5ee81bb4385..3d28fbf5520 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -38,7 +38,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) - libs := ctx.ImportedLibraries + libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/print_used_libraries_if_verbose.go b/legacy/builder/print_used_libraries_if_verbose.go index 8c32c925995..15211a46519 100644 --- a/legacy/builder/print_used_libraries_if_verbose.go +++ b/legacy/builder/print_used_libraries_if_verbose.go @@ -24,11 +24,11 @@ import ( type PrintUsedLibrariesIfVerbose struct{} func (s *PrintUsedLibrariesIfVerbose) Run(ctx *types.Context) error { - if !ctx.Verbose || len(ctx.ImportedLibraries) == 0 { + if !ctx.Verbose || len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { return nil } - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { legacy := "" if library.IsLegacy { legacy = tr("(legacy)") diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 621df16037d..95816e42a0e 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -137,7 +137,6 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.LibrariesManager = lm ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, - ctx.ImportedLibraries, ctx.Verbose, false, func(msg string) { ctx.Info(msg) }, diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index 127e953e2c0..ed6afc51d1e 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -44,7 +44,7 @@ func TestIncludesToIncludeFolders(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() require.Equal(t, 1, len(importedLibraries)) require.Equal(t, "Bridge", importedLibraries[0].Name) } @@ -67,7 +67,7 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() require.Equal(t, 0, len(importedLibraries)) } @@ -89,7 +89,7 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 2, len(importedLibraries)) require.Equal(t, "Bridge", importedLibraries[0].Name) @@ -114,7 +114,7 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 2, len(importedLibraries)) require.Equal(t, "ANewLibrary-master", importedLibraries[0].Name) @@ -144,7 +144,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 1, len(importedLibraries)) require.Equal(t, "SPI", importedLibraries[0].Name) @@ -175,7 +175,7 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 1, len(importedLibraries)) require.Equal(t, "SPI", importedLibraries[0].Name) @@ -206,7 +206,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 1, len(importedLibraries)) require.Equal(t, "USBHost", importedLibraries[0].Name) @@ -232,7 +232,7 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { NoError(t, err) } - importedLibraries := ctx.ImportedLibraries + importedLibraries := ctx.SketchLibrariesDetector.ImportedLibraries() sort.Sort(ByLibraryName(importedLibraries)) require.Equal(t, 3, len(importedLibraries)) require.Equal(t, "testlib1", importedLibraries[0].Name) diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index f6965cee0ff..5c04b1cee10 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -18,6 +18,7 @@ package test import ( "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -36,7 +37,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) @@ -56,7 +60,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err := cmd.Run(ctx) @@ -73,8 +80,10 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { NoError(t, temp.Join("dummy_file").WriteFile([]byte{})) ctx := &types.Context{} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{} cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index edf938877bb..beb4336fac8 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -25,7 +25,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" - "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -104,7 +103,6 @@ type Context struct { // Libraries handling LibrariesManager *librariesmanager.LibrariesManager - ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList diff --git a/legacy/builder/unused_compiled_libraries_remover.go b/legacy/builder/unused_compiled_libraries_remover.go index d1b36b147ac..03ea5b85d01 100644 --- a/legacy/builder/unused_compiled_libraries_remover.go +++ b/legacy/builder/unused_compiled_libraries_remover.go @@ -31,7 +31,7 @@ func (s *UnusedCompiledLibrariesRemover) Run(ctx *types.Context) error { return nil } - libraryNames := toLibraryNames(ctx.ImportedLibraries) + libraryNames := toLibraryNames(ctx.SketchLibrariesDetector.ImportedLibraries()) files, err := librariesBuildPath.ReadDir() if err != nil { diff --git a/legacy/builder/warn_about_arch_incompatible_libraries.go b/legacy/builder/warn_about_arch_incompatible_libraries.go index 712f79b2fe5..c815a54f3aa 100644 --- a/legacy/builder/warn_about_arch_incompatible_libraries.go +++ b/legacy/builder/warn_about_arch_incompatible_libraries.go @@ -33,7 +33,7 @@ func (s *WarnAboutArchIncompatibleLibraries) Run(ctx *types.Context) error { archs = append(archs, strings.Split(overrides, ",")...) } - for _, importedLibrary := range ctx.ImportedLibraries { + for _, importedLibrary := range ctx.SketchLibrariesDetector.ImportedLibraries() { if !importedLibrary.SupportsAnyArchitectureIn(archs...) { ctx.Info( tr("WARNING: library %[1]s claims to run on %[2]s architecture(s) and may be incompatible with your current board which runs on %[3]s architecture(s).", From 56b761147c236cd1301589ab1fba75337fa22c25 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 23:03:30 +0200 Subject: [PATCH 12/18] remove LibrariesResolutionResults from context --- arduino/builder/libraries.go | 33 +++++++++++ legacy/builder/builder.go | 5 +- .../print_used_and_not_used_libraries.go | 59 ------------------- legacy/builder/types/context.go | 5 +- legacy/builder/types/types.go | 5 -- 5 files changed, 39 insertions(+), 68 deletions(-) delete mode 100644 legacy/builder/print_used_and_not_used_libraries.go diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index d6f273f5f4c..2f8219c5241 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -18,6 +18,8 @@ package builder import ( "bytes" "fmt" + "strings" + "time" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" @@ -125,6 +127,37 @@ func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { return l.useCachedLibrariesResolution } +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + // AppendIncludeFolder todo should rename this, probably after refactoring the // container_find_includes command. //func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index f914e11140f..1af79855d5a 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -109,7 +109,10 @@ func (s *Builder) Run(ctx *types.Context) error { var otherErr error commands = []types.Command{ - &PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil}, + types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.PrintUsedAndNotUsedLibraries(mainErr != nil) + return nil + }), &PrintUsedLibrariesIfVerbose{}, diff --git a/legacy/builder/print_used_and_not_used_libraries.go b/legacy/builder/print_used_and_not_used_libraries.go deleted file mode 100644 index 971036bea45..00000000000 --- a/legacy/builder/print_used_and_not_used_libraries.go +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type PrintUsedAndNotUsedLibraries struct { - // Was there an error while compiling the sketch? - SketchError bool -} - -func (s *PrintUsedAndNotUsedLibraries) Run(ctx *types.Context) error { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !s.SketchError && !ctx.Verbose { - return nil - } - - res := "" - for header, libResResult := range ctx.LibrariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if s.SketchError { - ctx.Warn(res) - } else { - ctx.Info(res) - } - time.Sleep(100 * time.Millisecond) - return nil -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index beb4336fac8..6a50ecd21de 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -102,9 +102,8 @@ type Context struct { WarningsLevel string // Libraries handling - LibrariesManager *librariesmanager.LibrariesManager - LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeFolders paths.PathList + LibrariesManager *librariesmanager.LibrariesManager + IncludeFolders paths.PathList // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 2ebf92bae5a..d84864eecfe 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -95,11 +95,6 @@ func (f *SourceFile) DepfilePath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".d") } -type LibraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - type Command interface { Run(ctx *Context) error } From 34f52c34234291dd481544b49e352d8c1367d153 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 23:12:34 +0200 Subject: [PATCH 13/18] remove LibrariesManager from context --- commands/compile/compile.go | 14 +++++++------- legacy/builder/test/builder_test.go | 3 +-- legacy/builder/test/libraries_loader_test.go | 8 ++++---- legacy/builder/types/context.go | 2 -- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index c5f36dbcc7c..96f30a83f39 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -25,6 +25,7 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/buildcache" @@ -248,13 +249,13 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } } - // TODO replace all UseCache call with our SketchLibrariesDetector - useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + var libsManager *librariesmanager.LibrariesManager if pme.GetProfile() != nil { - builderCtx.LibrariesManager = lm + libsManager = lm } - lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( - useCachedLibrariesResolution, builderCtx.LibrariesManager, + useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + libsManager, libsResolver, verboseOut, err := bldr.LibrariesLoader( + useCachedLibrariesResolution, libsManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) @@ -262,13 +263,12 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return r, &arduino.CompileFailedError{Message: err.Error()} } - builderCtx.LibrariesManager = lm if builderCtx.Verbose { builderCtx.Warn(string(verboseOut)) } builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - lm, libsResolver, + libsManager, libsResolver, builderCtx.Verbose, useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 95816e42a0e..c4fc3a051b6 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -128,13 +128,12 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := bldr.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) NoError(t, err) - ctx.LibrariesManager = lm ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, ctx.Verbose, diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 66e27202fa5..dc4707455ff 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -48,7 +48,7 @@ func TestLoadLibrariesAVR(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -152,7 +152,7 @@ func TestLoadLibrariesSAM(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -229,7 +229,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -252,7 +252,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 6a50ecd21de..a78530387e9 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -25,7 +25,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -102,7 +101,6 @@ type Context struct { WarningsLevel string // Libraries handling - LibrariesManager *librariesmanager.LibrariesManager IncludeFolders paths.PathList // C++ Parsing From 57dc8d726eb3306b00b706462fc390d73ecc474a Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Sat, 2 Sep 2023 00:01:19 +0200 Subject: [PATCH 14/18] fix regression exceeding 100% status bar --- legacy/builder/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 1af79855d5a..8a8c8253d26 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -89,7 +89,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, } - ctx.Progress.AddSubSteps(len(commands) + 4) + ctx.Progress.AddSubSteps(len(commands) + 5) defer ctx.Progress.RemoveSubSteps() for _, command := range commands { From d97cac78848a82418b96ce9badb4abd0aab06f0e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Sat, 2 Sep 2023 21:21:39 +0200 Subject: [PATCH 15/18] refactor find_includes --- arduino/builder/detector/detector.go | 795 ++++++++++++++++++ arduino/builder/libraries.go | 240 ------ .../preprocessor/arduino_preprocessor.go | 2 +- arduino/builder/preprocessor/gcc.go | 2 +- arduino/builder/utils/utils.go | 185 ++++ commands/compile/compile.go | 8 +- legacy/builder/builder.go | 22 +- legacy/builder/builder_utils/utils.go | 123 +-- legacy/builder/container_find_includes.go | 474 ----------- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/includes_finder_with_regexp.go | 44 - legacy/builder/phases/core_builder.go | 2 +- legacy/builder/phases/libraries_builder.go | 5 +- legacy/builder/phases/sketch_builder.go | 5 +- legacy/builder/test/builder_test.go | 3 + .../unused_compiled_libraries_remover_test.go | 6 +- legacy/builder/types/accessories.go | 41 - legacy/builder/types/context.go | 6 +- legacy/builder/types/types.go | 80 -- legacy/builder/utils/utils.go | 61 -- 20 files changed, 1027 insertions(+), 1079 deletions(-) create mode 100644 arduino/builder/detector/detector.go delete mode 100644 arduino/builder/libraries.go create mode 100644 arduino/builder/utils/utils.go delete mode 100644 legacy/builder/container_find_includes.go delete mode 100644 legacy/builder/includes_finder_with_regexp.go delete mode 100644 legacy/builder/types/accessories.go diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go new file mode 100644 index 00000000000..aa00a02e718 --- /dev/null +++ b/arduino/builder/detector/detector.go @@ -0,0 +1,795 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "golang.org/x/exp/slices" + + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/arduino/libraries" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +var tr = i18n.Tr + +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + onlyUpdateCompilationDatabase bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + verboseStdoutFn func(data []byte) + verboseStderrFn func(data []byte) + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult + includeFolders paths.PathList +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + verbose, useCachedLibrariesResolution bool, + onlyUpdateCompilationDatabase bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), + verboseStdoutFn func(data []byte), + verboseStderrFn func(data []byte), +) *SketchLibrariesDetector { + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + verboseStdoutFn: verboseStdoutFn, + verboseStderrFn: verboseStderrFn, + importedLibraries: libraries.List{}, + includeFolders: paths.PathList{}, + onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) resolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +// ImportedLibraries todo +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// UseCachedLibrariesResolution todo +func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { + return l.useCachedLibrariesResolution +} + +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + +func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { + // TODO should we do a deep copy? + return l.includeFolders +} + +// AppendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +// Original comment: +// Append the given folder to the include path and match or append it to +// the cache. sourceFilePath and include indicate the source of this +// include (e.g. what #include line in what file it was resolved from) +// and should be the empty string for the default include folders, like +// the core or variant. +func (l *SketchLibrariesDetector) AppendIncludeFolder( + cache *includeCache, + sourceFilePath *paths.Path, + include string, + folder *paths.Path, +) { + l.includeFolders = append(l.includeFolders, folder) + cache.ExpectEntry(sourceFilePath, include, folder) +} + +// FindIncludes todo +func (l *SketchLibrariesDetector) FindIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + err := l.findIncludes(buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch) + if err != nil && l.onlyUpdateCompilationDatabase { + l.verboseInfoFn( + fmt.Sprintf( + "%s: %s", + tr("An error occurred detecting libraries"), + tr("the compilation database may be incomplete or inaccurate"), + ), + ) + return nil + } + return err +} + +func (l *SketchLibrariesDetector) findIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + librariesResolutionCache := buildPath.Join("libraries.cache") + if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { + if d, err := librariesResolutionCache.ReadFile(); err != nil { + return err + } else { + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { + return err + } + } + if l.verbose { + l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) + } + return nil + } + + cachePath := buildPath.Join("includes.cache") + cache := readCache(cachePath) + + l.AppendIncludeFolder(cache, nil, "", buildCorePath) + if buildVariantPath != nil { + l.AppendIncludeFolder(cache, nil, "", buildVariantPath) + } + + sourceFileQueue := &UniqueSourceFileQueue{} + + if !l.useCachedLibrariesResolution { + sketch := sketch + mergedfile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(mergedfile) + + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) + srcSubfolderPath := sketchBuildPath.Join("src") + if srcSubfolderPath.IsDir() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) + } + + for !sourceFileQueue.Empty() { + err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) + if err != nil { + cachePath.Remove() + return errors.WithStack(err) + } + } + + // Finalize the cache + cache.ExpectEnd() + if err := writeCache(cache, cachePath); err != nil { + return errors.WithStack(err) + } + } + + if err := l.failIfImportedLibraryIsWrong(); err != nil { + return errors.WithStack(err) + } + + if d, err := json.Marshal(l.includeFolders); err != nil { + return err + } else if err := librariesResolutionCache.WriteFile(d); err != nil { + return err + } + + return nil +} + +func (l *SketchLibrariesDetector) findIncludesUntilDone( + cache *includeCache, + sourceFileQueue *UniqueSourceFileQueue, + buildProperties *properties.Map, + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + platformArch string, +) error { + sourceFile := sourceFileQueue.Pop() + sourcePath := sourceFile.SourcePath() + targetFilePath := paths.NullPath() + depPath := sourceFile.DepfilePath() + objPath := sourceFile.ObjectPath() + + // TODO: This should perhaps also compare against the + // include.cache file timestamp. Now, it only checks if the file + // changed after the object file was generated, but if it + // changed between generating the cache and the object file, + // this could show the file as unchanged when it really is + // changed. Changing files during a build isn't really + // supported, but any problems from it should at least be + // resolved when doing another build, which is not currently the + // case. + // TODO: This reads the dependency file, but the actual building + // does it again. Should the result be somehow cached? Perhaps + // remove the object file if it is found to be stale? + unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) + if err != nil { + return errors.WithStack(err) + } + + first := true + for { + cache.ExpectFile(sourcePath) + + // Libraries may require the "utility" directory to be added to the include + // search path, but only for the source code of the library, so we temporary + // copy the current search path list and add the library' utility directory + // if needed. + includeFolders := l.includeFolders + if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { + includeFolders = append(includeFolders, extraInclude) + } + + var preprocErr error + var preprocStderr []byte + + var missingIncludeH string + if unchanged && cache.valid { + missingIncludeH = cache.Next().Include + if first && l.verbose { + l.verboseInfoFn(tr("Using cached library dependencies for file: %[1]s", sourcePath)) + } + } else { + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + // Unwrap error and see if it is an ExitError. + if preprocErr == nil { + // Preprocessor successful, done + missingIncludeH = "" + } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { + // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors + return errors.WithStack(preprocErr) + } else { + missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) + if missingIncludeH == "" && l.verbose { + l.verboseInfoFn(tr("Error while detecting libraries included by %[1]s", sourcePath)) + } + } + } + + if missingIncludeH == "" { + // No missing includes found, we're done + cache.ExpectEntry(sourcePath, "", nil) + return nil + } + + library := l.resolveLibrary(missingIncludeH, platformArch) + if library == nil { + // Library could not be resolved, show error + if preprocErr == nil || preprocStderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + if preprocErr == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New(tr("Internal error in cache")) + } + } + l.verboseStderrFn(preprocStderr) + return errors.WithStack(preprocErr) + } + + // Add this library to the list of libraries, the + // include path and queue its source files for further + // include scanning + l.AppendImportedLibraries(library) + l.AppendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + + if library.Precompiled && library.PrecompiledWithSources { + // Fully precompiled libraries should have no dependencies to avoid ABI breakage + if l.verbose { + l.verboseInfoFn(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) + } + } else { + for _, sourceDir := range library.SourceDirs() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) + } + } + first = false + } +} + +func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + sourceFileQueue *UniqueSourceFileQueue, + origin interface{}, + folder *paths.Path, + recurse bool, +) error { + sourceFileExtensions := []string{} + for k := range globals.SourceFilesValidExtensions { + sourceFileExtensions = append(sourceFileExtensions, k) + } + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) + if err != nil { + return errors.WithStack(err) + } + + for _, filePath := range filePaths { + sourceFile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(sourceFile) + } + + return nil +} + +func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { + if len(l.importedLibraries) == 0 { + return nil + } + + for _, library := range l.importedLibraries { + if !library.IsLegacy { + if library.InstallDir.Join("arch").IsDir() { + return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) + } + for _, propName := range libraries.MandatoryProperties { + if !library.Properties.ContainsKey(propName) { + return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) + } + } + if library.Layout == libraries.RecursiveLayout { + if library.UtilityDir != nil { + return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) + } + } + } + } + + return nil +} + +var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") + +func IncludesFinderWithRegExp(source string) string { + match := INCLUDE_REGEXP.FindStringSubmatch(source) + if match != nil { + return strings.TrimSpace(match[1]) + } + return findIncludeForOldCompilers(source) +} + +func findIncludeForOldCompilers(source string) string { + lines := strings.Split(source, "\n") + for _, line := range lines { + splittedLine := strings.Split(line, ":") + for i := range splittedLine { + if strings.Contains(splittedLine[i], "fatal error") { + return strings.TrimSpace(splittedLine[i+1]) + } + } + } + return "" +} + +type SourceFile struct { + // Path to the source file within the sketch/library root folder + relativePath *paths.Path + + // ExtraIncludePath contains an extra include path that must be + // used to compile this source file. + // This is mainly used for source files that comes from old-style libraries + // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. + extraIncludePath *paths.Path + + // The source root for the given origin, where its source files + // can be found. Prepending this to SourceFile.RelativePath will give + // the full path to that source file. + sourceRoot *paths.Path + + // The build root for the given origin, where build products will + // be placed. Any directories inside SourceFile.RelativePath will be + // appended here. + buildRoot *paths.Path +} + +func (f *SourceFile) Equals(g *SourceFile) bool { + return f.relativePath.EqualsTo(g.relativePath) && + f.buildRoot.EqualsTo(g.buildRoot) && + f.sourceRoot.EqualsTo(g.sourceRoot) +} + +// Create a SourceFile containing the given source file path within the +// given origin. The given path can be absolute, or relative within the +// origin's root source folder +func MakeSourceFile( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + origin interface{}, + path *paths.Path, +) (*SourceFile, error) { + res := &SourceFile{} + + switch o := origin.(type) { + case *sketch.Sketch: + res.buildRoot = sketchBuildPath + res.sourceRoot = sketchBuildPath + case *libraries.Library: + res.buildRoot = librariesBuildPath.Join(o.DirName) + res.sourceRoot = o.SourceDir + res.extraIncludePath = o.UtilityDir + default: + panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) + } + + if path.IsAbs() { + var err error + path, err = res.sourceRoot.RelTo(path) + if err != nil { + return nil, err + } + } + res.relativePath = path + return res, nil +} + +func (f *SourceFile) ExtraIncludePath() *paths.Path { + return f.extraIncludePath +} + +func (f *SourceFile) SourcePath() *paths.Path { + return f.sourceRoot.JoinPath(f.relativePath) +} + +func (f *SourceFile) ObjectPath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".o") +} + +func (f *SourceFile) DepfilePath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".d") +} + +// LibrariesLoader todo +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { + // Since we are using the cached libraries resolution + // the library manager is not needed. + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) + + builtInLibrariesFolders := builtInLibrariesDirs + if builtInLibrariesFolders != nil { + if err := builtInLibrariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) + } + + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) + } + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) + + librariesFolders := otherLibrariesDirs + if err := librariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + for _, folder := range librariesFolders { + lm.AddLibrariesDir(folder, libraries.User) + } + + for _, status := range lm.RescanLibraries() { + // With the refactoring of the initialization step of the CLI we changed how + // errors are returned when loading platforms and libraries, that meant returning a list of + // errors instead of a single one to enhance the experience for the user. + // I have no intention right now to start a refactoring of the legacy package too, so + // here's this shitty solution for now. + // When we're gonna refactor the legacy package this will be gone. + verboseOut.Write([]byte(status.Message())) + } + + for _, dir := range libraryDirs { + // Libraries specified this way have top priority + if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + } + + resolver := librariesresolver.NewCppResolver() + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + return lm, resolver, verboseOut.Bytes(), nil +} + +func (l *SketchLibrariesDetector) OnlyUpdateCompilationDatabase() bool { + return l.onlyUpdateCompilationDatabase +} + +type includeCacheEntry struct { + Sourcefile *paths.Path + Include string + Includepath *paths.Path +} + +func (entry *includeCacheEntry) String() string { + return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", + entry.Sourcefile, entry.Include, entry.Includepath) +} + +func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { + return entry.String() == other.String() +} + +type includeCache struct { + // Are the cache contents valid so far? + valid bool + // Index into entries of the next entry to be processed. Unused + // when the cache is invalid. + next int + entries []*includeCacheEntry +} + +// Return the next cache entry. Should only be called when the cache is +// valid and a next entry is available (the latter can be checked with +// ExpectFile). Does not advance the cache. +func (cache *includeCache) Next() *includeCacheEntry { + return cache.entries[cache.next] +} + +// Check that the next cache entry is about the given file. If it is +// not, or no entry is available, the cache is invalidated. Does not +// advance the cache. +func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { + if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Check that the next entry matches the given values. If so, advance +// the cache. If not, the cache is invalidated. If the cache is +// invalidated, or was already invalid, an entry with the given values +// is appended. +func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { + entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} + if cache.valid { + if cache.next < len(cache.entries) && cache.Next().Equals(entry) { + cache.next++ + } else { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } + } + + if !cache.valid { + cache.entries = append(cache.entries, entry) + } +} + +// Check that the cache is completely consumed. If not, the cache is +// invalidated. +func (cache *includeCache) ExpectEnd() { + if cache.valid && cache.next < len(cache.entries) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Read the cache from the given file +func readCache(path *paths.Path) *includeCache { + bytes, err := path.ReadFile() + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result := &includeCache{} + err = json.Unmarshal(bytes, &result.entries) + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result.valid = true + return result +} + +// Write the given cache to the given file if it is invalidated. If the +// cache is still valid, just update the timestamps of the file. +func writeCache(cache *includeCache, path *paths.Path) error { + // If the cache was still valid all the way, just touch its file + // (in case any source file changed without influencing the + // includes). If it was invalidated, overwrite the cache with + // the new contents. + if cache.valid { + path.Chtimes(time.Now(), time.Now()) + } else { + bytes, err := json.MarshalIndent(cache.entries, "", " ") + if err != nil { + return errors.WithStack(err) + } + err = path.WriteFile(bytes) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +type UniqueSourceFileQueue []*SourceFile + +func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { + if !queue.Contains(value) { + *queue = append(*queue, value) + } +} + +func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { + return slices.ContainsFunc(queue, target.Equals) +} + +func (queue *UniqueSourceFileQueue) Pop() *SourceFile { + old := *queue + x := old[0] + *queue = old[1:] + return x +} + +func (queue UniqueSourceFileQueue) Empty() bool { + return len(queue) == 0 +} diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go deleted file mode 100644 index 2f8219c5241..00000000000 --- a/arduino/builder/libraries.go +++ /dev/null @@ -1,240 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/arduino/cores" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type libraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - -// SketchLibrariesDetector todo -type SketchLibrariesDetector struct { - librariesManager *librariesmanager.LibrariesManager - librariesResolver *librariesresolver.Cpp - useCachedLibrariesResolution bool - verbose bool - verboseInfoFn func(msg string) - verboseWarnFn func(msg string) - importedLibraries libraries.List - librariesResolutionResults map[string]libraryResolutionResult -} - -// NewSketchLibrariesDetector todo -func NewSketchLibrariesDetector( - lm *librariesmanager.LibrariesManager, - libsResolver *librariesresolver.Cpp, - verbose, useCachedLibrariesResolution bool, - verboseInfoFn func(msg string), - verboseWarnFn func(msg string), -) *SketchLibrariesDetector { - return &SketchLibrariesDetector{ - librariesManager: lm, - librariesResolver: libsResolver, - useCachedLibrariesResolution: useCachedLibrariesResolution, - librariesResolutionResults: map[string]libraryResolutionResult{}, - verbose: verbose, - verboseInfoFn: verboseInfoFn, - verboseWarnFn: verboseWarnFn, - importedLibraries: libraries.List{}, - } -} - -// ResolveLibrary todo -func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *libraries.Library { - importedLibraries := l.importedLibraries - candidates := l.librariesResolver.AlternativesFor(header) - - if l.verbose { - l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) - l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := l.librariesResolver.ResolveFor(header, platformArch) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - l.librariesResolutionResults[header] = libraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected -} - -// ImportedLibraries todo -func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { - // TODO understand if we have to do a deepcopy - return l.importedLibraries -} - -// AppendImportedLibraries todo should rename this, probably after refactoring the -// container_find_includes command. -func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { - l.importedLibraries = append(l.importedLibraries, library) -} - -// UseCachedLibrariesResolution todo -func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { - return l.useCachedLibrariesResolution -} - -// PrintUsedAndNotUsedLibraries todo -func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !sketchError && !l.verbose { - return - } - - res := "" - for header, libResResult := range l.librariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if sketchError { - l.verboseWarnFn(res) - } else { - l.verboseInfoFn(res) - } - // todo why?? should we remove this? - time.Sleep(100 * time.Millisecond) -} - -// AppendIncludeFolder todo should rename this, probably after refactoring the -// container_find_includes command. -//func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { -// ctx.IncludeFolders = append(ctx.IncludeFolders, folder) -// cache.ExpectEntry(sourceFilePath, include, folder) -//} - -// LibrariesLoader todo -func LibrariesLoader( - useCachedLibrariesResolution bool, - librariesManager *librariesmanager.LibrariesManager, - builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, - actualPlatform, targetPlatform *cores.PlatformRelease, -) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { - verboseOut := &bytes.Buffer{} - lm := librariesManager - if useCachedLibrariesResolution { - // Since we are using the cached libraries resolution - // the library manager is not needed. - lm = librariesmanager.NewLibraryManager(nil, nil) - } - if librariesManager == nil { - lm = librariesmanager.NewLibraryManager(nil, nil) - - builtInLibrariesFolders := builtInLibrariesDirs - if builtInLibrariesFolders != nil { - if err := builtInLibrariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) - } - - if actualPlatform != targetPlatform { - lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) - } - lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) - - librariesFolders := otherLibrariesDirs - if err := librariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - for _, folder := range librariesFolders { - lm.AddLibrariesDir(folder, libraries.User) - } - - for _, status := range lm.RescanLibraries() { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. - verboseOut.Write([]byte(status.Message())) - } - - for _, dir := range libraryDirs { - // Libraries specified this way have top priority - if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - } - - resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if actualPlatform != targetPlatform { - if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - return lm, resolver, verboseOut.Bytes(), nil -} diff --git a/arduino/builder/preprocessor/arduino_preprocessor.go b/arduino/builder/preprocessor/arduino_preprocessor.go index 8629ec211ba..a81d4bff612 100644 --- a/arduino/builder/preprocessor/arduino_preprocessor.go +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -21,9 +21,9 @@ import ( "path/filepath" "runtime" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 476c1f2f5b4..9f63eaa93df 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go new file mode 100644 index 00000000000..64c1a54dbdd --- /dev/null +++ b/arduino/builder/utils/utils.go @@ -0,0 +1,185 @@ +package utils + +import ( + "os" + "strings" + "unicode" + + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { + logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) + if objectFile == nil || dependencyFile == nil { + logrus.Debugf("Not found: nil") + return false, nil + } + + sourceFile = sourceFile.Clean() + sourceFileStat, err := sourceFile.Stat() + if err != nil { + return false, errors.WithStack(err) + } + + objectFile = objectFile.Clean() + objectFileStat, err := objectFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", objectFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + dependencyFile = dependencyFile.Clean() + dependencyFileStat, err := dependencyFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", dependencyFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, objectFile) + return false, nil + } + if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) + return false, nil + } + + rows, err := dependencyFile.ReadFileAsLines() + if err != nil { + return false, errors.WithStack(err) + } + + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) + + if len(rows) == 0 { + return true, nil + } + + firstRow := rows[0] + if !strings.HasSuffix(firstRow, ":") { + logrus.Debugf("No colon in first line of depfile") + return false, nil + } + objFileInDepFile := firstRow[:len(firstRow)-1] + if objFileInDepFile != objectFile.String() { + logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) + return false, nil + } + + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + return false, nil + } + + rows = rows[1:] + for _, row := range rows { + depStat, err := os.Stat(row) + if err != nil && !os.IsNotExist(err) { + // There is probably a parsing error of the dep file + // Ignore the error and trigger a full rebuild anyway + logrus.WithError(err).Debugf("Failed to read: %v", row) + return false, nil + } + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", row) + return false, nil + } + if depStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", row, objectFile) + return false, nil + } + } + + return true, nil +} + +func removeEndingBackSlash(s string) string { + return strings.TrimSuffix(s, "\\") +} + +func unescapeDep(s string) string { + s = strings.ReplaceAll(s, "\\ ", " ") + s = strings.ReplaceAll(s, "\\\t", "\t") + s = strings.ReplaceAll(s, "\\#", "#") + s = strings.ReplaceAll(s, "$$", "$") + s = strings.ReplaceAll(s, "\\\\", "\\") + return s +} + +// Normalizes an UTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + +var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} + +// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func FilterOutSCCS(file *paths.Path) bool { + return !SOURCE_CONTROL_FOLDERS[file.Base()] +} + +// FilterReadableFiles is a ReadDirFilter that accepts only readable files +func FilterReadableFiles(file *paths.Path) bool { + // See if the file is readable by opening it + f, err := file.Open() + if err != nil { + return false + } + f.Close() + return true +} + +// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") + +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { + fileFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + paths.FilterOutDirectories(), + FilterReadableFiles, + ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } + if recurse { + dirFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + ) + return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) + } + return dir.ReadDir(fileFilter) +} + +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 96f30a83f39..1daa6f31714 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -24,6 +24,7 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -254,7 +255,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream libsManager = lm } useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() - libsManager, libsResolver, verboseOut, err := bldr.LibrariesLoader( + libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( useCachedLibrariesResolution, libsManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, @@ -267,12 +268,15 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Warn(string(verboseOut)) } - builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + builderCtx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( libsManager, libsResolver, builderCtx.Verbose, useCachedLibrariesResolution, + req.GetCreateCompilationDatabaseOnly(), func(msg string) { builderCtx.Info(msg) }, func(msg string) { builderCtx.Warn(msg) }, + func(data []byte) { builderCtx.WriteStdout(data) }, + func(data []byte) { builderCtx.WriteStderr(data) }, ) defer func() { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 8a8c8253d26..c9336d24528 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -52,7 +52,7 @@ func (s *Builder) Run(ctx *types.Context) error { }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -144,7 +144,7 @@ func PreprocessSketch(ctx *types.Context) error { preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor } normalOutput, verboseOutput, err := preprocessorImpl( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) if ctx.Verbose { ctx.WriteStdout(verboseOutput) @@ -172,7 +172,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { return _err }), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -220,3 +220,19 @@ func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) } + +func findIncludes(ctx *types.Context) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + return nil + }) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index f7cd0348c55..a827e496dac 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -24,16 +24,15 @@ import ( "strings" "sync" + bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/i18n" - f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) var tr = i18n.Tr @@ -48,7 +47,7 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } targetModTime := targetStat.ModTime() - files, err := utils.FindFilesInFolder(dir, true, extensions...) + files, err := bUtils.FindFilesInFolder(dir, true, extensions...) if err != nil { return false, err } @@ -78,7 +77,7 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build validExtensions = append(validExtensions, ext) } - sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) if err != nil { return nil, err } @@ -169,7 +168,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { return nil, errors.WithStack(err) } @@ -205,120 +204,6 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return objectFile, nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { - logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) - if objectFile == nil || dependencyFile == nil { - logrus.Debugf("Not found: nil") - return false, nil - } - - sourceFile = sourceFile.Clean() - sourceFileStat, err := sourceFile.Stat() - if err != nil { - return false, errors.WithStack(err) - } - - objectFile = objectFile.Clean() - objectFileStat, err := objectFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", objectFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - dependencyFile = dependencyFile.Clean() - dependencyFileStat, err := dependencyFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", dependencyFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, objectFile) - return false, nil - } - if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) - return false, nil - } - - rows, err := dependencyFile.ReadFileAsLines() - if err != nil { - return false, errors.WithStack(err) - } - - rows = f.Map(rows, removeEndingBackSlash) - rows = f.Map(rows, strings.TrimSpace) - rows = f.Map(rows, unescapeDep) - rows = f.Filter(rows, f.NotEquals("")) - - if len(rows) == 0 { - return true, nil - } - - firstRow := rows[0] - if !strings.HasSuffix(firstRow, ":") { - logrus.Debugf("No colon in first line of depfile") - return false, nil - } - objFileInDepFile := firstRow[:len(firstRow)-1] - if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) - return false, nil - } - - // The first line of the depfile contains the path to the object file to generate. - // The second line of the depfile contains the path to the source file. - // All subsequent lines contain the header files necessary to compile the object file. - - // If we don't do this check it might happen that trying to compile a source file - // that has the same name but a different path wouldn't recreate the object file. - if sourceFile.String() != strings.Trim(rows[1], " ") { - return false, nil - } - - rows = rows[1:] - for _, row := range rows { - depStat, err := os.Stat(row) - if err != nil && !os.IsNotExist(err) { - // There is probably a parsing error of the dep file - // Ignore the error and trigger a full rebuild anyway - logrus.WithError(err).Debugf("Failed to read: %v", row) - return false, nil - } - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", row) - return false, nil - } - if depStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", row, objectFile) - return false, nil - } - } - - return true, nil -} - -func unescapeDep(s string) string { - s = strings.Replace(s, "\\ ", " ", -1) - s = strings.Replace(s, "\\\t", "\t", -1) - s = strings.Replace(s, "\\#", "#", -1) - s = strings.Replace(s, "$$", "$", -1) - s = strings.Replace(s, "\\\\", "\\", -1) - return s -} - -func removeEndingBackSlash(s string) string { - return strings.TrimSuffix(s, "\\") -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go deleted file mode 100644 index b90c25d46d5..00000000000 --- a/legacy/builder/container_find_includes.go +++ /dev/null @@ -1,474 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -/* - -Include detection - -This code is responsible for figuring out what libraries the current -sketch needs an populates both Context.ImportedLibraries with a list of -Library objects, and Context.IncludeFolders with a list of folders to -put on the include path. - -Simply put, every #include in a source file pulls in the library that -provides that source file. This includes source files in the selected -libraries, so libraries can recursively include other libraries as well. - -To implement this, the gcc preprocessor is used. A queue is created -containing, at first, the source files in the sketch. Each of the files -in the queue is processed in turn by running the preprocessor on it. If -the preprocessor provides an error, the output is examined to see if the -error is a missing header file originating from a #include directive. - -The filename is extracted from that #include directive, and a library is -found that provides it. If multiple libraries provide the same file, one -is slected (how this selection works is not described here, see the -ResolveLibrary function for that). The library selected in this way is -added to the include path through Context.IncludeFolders and the list of -libraries to include in the link through Context.ImportedLibraries. - -Furthermore, all of the library source files are added to the queue, to -be processed as well. When the preprocessor completes without showing an -#include error, processing of the file is complete and it advances to -the next. When no library can be found for a included filename, an error -is shown and the process is aborted. - -Caching - -Since this process is fairly slow (requiring at least one invocation of -the preprocessor per source file), its results are cached. - -Just caching the complete result (i.e. the resulting list of imported -libraries) seems obvious, but such a cache is hard to invalidate. Making -a list of all the source and header files used to create the list and -check if any of them changed is probably feasible, but this would also -require caching the full list of libraries to invalidate the cache when -the include to library resolution might have a different result. Another -downside of a complete cache is that any changes requires re-running -everything, even if no includes were actually changed. - -Instead, caching happens by keeping a sort of "journal" of the steps in -the include detection, essentially tracing each file processed and each -include path entry added. The cache is used by retracing these steps: -The include detection process is executed normally, except that instead -of running the preprocessor, the include filenames are (when possible) -read from the cache. Then, the include file to library resolution is -again executed normally. The results are checked against the cache and -as long as the results match, the cache is considered valid. - -When a source file (or any of the files it includes, as indicated by the -.d file) is changed, the preprocessor is executed as normal for the -file, ignoring any includes from the cache. This does not, however, -invalidate the cache: If the results from the preprocessor match the -entries in the cache, the cache remains valid and can again be used for -the next (unchanged) file. - -The cache file uses the JSON format and contains a list of entries. Each -entry represents a discovered library and contains: - - Sourcefile: The source file that the include was found in - - Include: The included filename found - - Includepath: The addition to the include path - -There are also some special entries: - - When adding the initial include path entries, such as for the core - and variant paths. These are not discovered, so the Sourcefile and - Include fields will be empty. - - When a file contains no (more) missing includes, an entry with an - empty Include and IncludePath is generated. - -*/ - -package builder - -import ( - "encoding/json" - "fmt" - "os/exec" - "time" - - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ContainerFindIncludes struct{} - -func (s *ContainerFindIncludes) Run(ctx *types.Context) error { - err := s.findIncludes(ctx) - if err != nil && ctx.OnlyUpdateCompilationDatabase { - ctx.Info( - fmt.Sprintf("%s: %s", - tr("An error occurred detecting libraries"), - tr("the compilation database may be incomplete or inaccurate"))) - return nil - } - return err -} - -func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { - librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { - return err - } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { - return err - } - if ctx.Verbose { - ctx.Info("Using cached library discovery: " + librariesResolutionCache.String()) - } - return nil - } - - cachePath := ctx.BuildPath.Join("includes.cache") - cache := readCache(cachePath) - - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.core.path")) - if ctx.BuildProperties.Get("build.variant.path") != "" { - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) - } - - sourceFileQueue := &types.UniqueSourceFileQueue{} - - if !ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() { - sketch := ctx.Sketch - mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(mergedfile) - - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */) - srcSubfolderPath := ctx.SketchBuildPath.Join("src") - if srcSubfolderPath.IsDir() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) - } - - for !sourceFileQueue.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFileQueue) - if err != nil { - cachePath.Remove() - return errors.WithStack(err) - } - } - - // Finalize the cache - cache.ExpectEnd() - if err := writeCache(cache, cachePath); err != nil { - return errors.WithStack(err) - } - } - - if err := failIfImportedLibraryIsWrong(ctx); err != nil { - return errors.WithStack(err) - } - - if d, err := json.Marshal(ctx.IncludeFolders); err != nil { - return err - } else if err := librariesResolutionCache.WriteFile(d); err != nil { - return err - } - - return nil -} - -// Append the given folder to the include path and match or append it to -// the cache. sourceFilePath and include indicate the source of this -// include (e.g. what #include line in what file it was resolved from) -// and should be the empty string for the default include folders, like -// the core or variant. -func appendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { - ctx.IncludeFolders = append(ctx.IncludeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) -} - -type includeCacheEntry struct { - Sourcefile *paths.Path - Include string - Includepath *paths.Path -} - -func (entry *includeCacheEntry) String() string { - return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", - entry.Sourcefile, entry.Include, entry.Includepath) -} - -func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { - return entry.String() == other.String() -} - -type includeCache struct { - // Are the cache contents valid so far? - valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// Check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// Check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Read the cache from the given file -func readCache(path *paths.Path) *includeCache { - bytes, err := path.ReadFile() - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result.valid = true - return result -} - -// Write the given cache to the given file if it is invalidated. If the -// cache is still valid, just update the timestamps of the file. -func writeCache(cache *includeCache, path *paths.Path) error { - // If the cache was still valid all the way, just touch its file - // (in case any source file changed without influencing the - // includes). If it was invalidated, overwrite the cache with - // the new contents. - if cache.valid { - path.Chtimes(time.Now(), time.Now()) - } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") - if err != nil { - return errors.WithStack(err) - } - err = path.WriteFile(bytes) - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - -func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error { - sourceFile := sourceFileQueue.Pop() - sourcePath := sourceFile.SourcePath() - targetFilePath := paths.NullPath() - depPath := sourceFile.DepfilePath() - objPath := sourceFile.ObjectPath() - - // TODO: This should perhaps also compare against the - // include.cache file timestamp. Now, it only checks if the file - // changed after the object file was generated, but if it - // changed between generating the cache and the object file, - // this could show the file as unchanged when it really is - // changed. Changing files during a build isn't really - // supported, but any problems from it should at least be - // resolved when doing another build, which is not currently the - // case. - // TODO: This reads the dependency file, but the actual building - // does it again. Should the result be somehow cached? Perhaps - // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) - if err != nil { - return errors.WithStack(err) - } - - first := true - for { - cache.ExpectFile(sourcePath) - - // Libraries may require the "utility" directory to be added to the include - // search path, but only for the source code of the library, so we temporary - // copy the current search path list and add the library' utility directory - // if needed. - includeFolders := ctx.IncludeFolders - if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { - includeFolders = append(includeFolders, extraInclude) - } - - var preprocErr error - var preprocStderr []byte - - var missingIncludeH string - if unchanged && cache.valid { - missingIncludeH = cache.Next().Include - if first && ctx.Verbose { - ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) - } - } else { - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - // Unwrap error and see if it is an ExitError. - if preprocErr == nil { - // Preprocessor successful, done - missingIncludeH = "" - } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { - // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors - return errors.WithStack(preprocErr) - } else { - missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) - if missingIncludeH == "" && ctx.Verbose { - ctx.Info(tr("Error while detecting libraries included by %[1]s", sourcePath)) - } - } - } - - if missingIncludeH == "" { - // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) - return nil - } - - library := ResolveLibrary(ctx, missingIncludeH) - if library == nil { - // Library could not be resolved, show error - if preprocErr == nil || preprocStderr == nil { - // Filename came from cache, so run preprocessor to obtain error to show - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - if preprocErr == nil { - // If there is a missing #include in the cache, but running - // gcc does not reproduce that, there is something wrong. - // Returning an error here will cause the cache to be - // deleted, so hopefully the next compilation will succeed. - return errors.New(tr("Internal error in cache")) - } - } - ctx.WriteStderr(preprocStderr) - return errors.WithStack(preprocErr) - } - - // Add this library to the list of libraries, the - // include path and queue its source files for further - // include scanning - ctx.SketchLibrariesDetector.AppendImportedLibraries(library) - appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) - - if library.Precompiled && library.PrecompiledWithSources { - // Fully precompiled libraries should have no dependencies to avoid ABI breakage - if ctx.Verbose { - ctx.Info(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) - } - } else { - for _, sourceDir := range library.SourceDirs() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) - } - } - first = false - } -} - -func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { - sourceFileExtensions := []string{} - for k := range globals.SourceFilesValidExtensions { - sourceFileExtensions = append(sourceFileExtensions, k) - } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) - if err != nil { - return errors.WithStack(err) - } - - for _, filePath := range filePaths { - sourceFile, err := types.MakeSourceFile(ctx, origin, filePath) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(sourceFile) - } - - return nil -} - -func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - return ctx.SketchLibrariesDetector.ResolveLibrary(header, ctx.TargetPlatform.Platform.Architecture) -} - -func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { - return nil - } - - for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { - if !library.IsLegacy { - if library.InstallDir.Join("arch").IsDir() { - return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) - } - for _, propName := range libraries.MandatoryProperties { - if !library.Properties.ContainsKey(propName) { - return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) - } - } - if library.Layout == libraries.RecursiveLayout { - if library.UtilityDir != nil { - return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) - } - } - } - } - - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 37b60e92fac..45f68ef9ed4 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -26,11 +26,11 @@ import ( properties "github.com/arduino/go-properties-orderedmap" "golang.org/x/exp/slices" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" ) type ExportProjectCMake struct { diff --git a/legacy/builder/includes_finder_with_regexp.go b/legacy/builder/includes_finder_with_regexp.go deleted file mode 100644 index 03017c67048..00000000000 --- a/legacy/builder/includes_finder_with_regexp.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "regexp" - "strings" -) - -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") - -func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) - if match != nil { - return strings.TrimSpace(match[1]) - } - return findIncludeForOldCompilers(source) -} - -func findIncludeForOldCompilers(source string) string { - lines := strings.Split(source, "\n") - for _, line := range lines { - splittedLine := strings.Split(line, ":") - for i := range splittedLine { - if strings.Contains(splittedLine[i], "fatal error") { - return strings.TrimSpace(splittedLine[i+1]) - } - } - } - return "" -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 5c94573f349..455588f7444 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -28,7 +28,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 3d28fbf5520..88106c9a9de 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,12 +18,12 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -37,7 +37,8 @@ type LibrariesBuilder struct{} func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 300cb6836af..398ba87de8b 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,10 +16,10 @@ package phases import ( + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" ) @@ -28,7 +28,8 @@ type SketchBuilder struct{} func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index c4fc3a051b6..10b4f4a4a24 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -138,8 +138,11 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat lm, libsResolver, ctx.Verbose, false, + false, func(msg string) { ctx.Info(msg) }, func(msg string) { ctx.Warn(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, ) } diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index 5c04b1cee10..ab49dc9e26f 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -38,7 +38,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -61,7 +61,7 @@ func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -81,7 +81,7 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { ctx := &types.Context{} ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.LibrariesBuildPath = temp diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go deleted file mode 100644 index b4de7a1a18a..00000000000 --- a/legacy/builder/types/accessories.go +++ /dev/null @@ -1,41 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package types - -import "golang.org/x/exp/slices" - -type UniqueSourceFileQueue []*SourceFile - -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { - *queue = append(*queue, value) - } -} - -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { - return slices.ContainsFunc(queue, target.Equals) -} - -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue UniqueSourceFileQueue) Empty() bool { - return len(queue) == 0 -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index a78530387e9..a6570bf3268 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -62,7 +63,7 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { Builder *builder.Builder - SketchLibrariesDetector *builder.SketchLibrariesDetector + SketchLibrariesDetector *detector.SketchLibrariesDetector // Build options HardwareDirs paths.PathList @@ -100,9 +101,6 @@ type Context struct { Sketch *sketch.Sketch WarningsLevel string - // Libraries handling - IncludeFolders paths.PathList - // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index d84864eecfe..8678fd2fe51 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -15,86 +15,6 @@ package types -import ( - "fmt" - - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/sketch" - paths "github.com/arduino/go-paths-helper" -) - -type SourceFile struct { - // Path to the source file within the sketch/library root folder - relativePath *paths.Path - - // ExtraIncludePath contains an extra include path that must be - // used to compile this source file. - // This is mainly used for source files that comes from old-style libraries - // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. - extraIncludePath *paths.Path - - // The source root for the given origin, where its source files - // can be found. Prepending this to SourceFile.RelativePath will give - // the full path to that source file. - sourceRoot *paths.Path - - // The build root for the given origin, where build products will - // be placed. Any directories inside SourceFile.RelativePath will be - // appended here. - buildRoot *paths.Path -} - -func (f *SourceFile) Equals(g *SourceFile) bool { - return f.relativePath.EqualsTo(g.relativePath) && - f.buildRoot.EqualsTo(g.buildRoot) && - f.sourceRoot.EqualsTo(g.sourceRoot) -} - -// Create a SourceFile containing the given source file path within the -// given origin. The given path can be absolute, or relative within the -// origin's root source folder -func MakeSourceFile(ctx *Context, origin interface{}, path *paths.Path) (*SourceFile, error) { - res := &SourceFile{} - - switch o := origin.(type) { - case *sketch.Sketch: - res.buildRoot = ctx.SketchBuildPath - res.sourceRoot = ctx.SketchBuildPath - case *libraries.Library: - res.buildRoot = ctx.LibrariesBuildPath.Join(o.DirName) - res.sourceRoot = o.SourceDir - res.extraIncludePath = o.UtilityDir - default: - panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) - } - - if path.IsAbs() { - var err error - path, err = res.sourceRoot.RelTo(path) - if err != nil { - return nil, err - } - } - res.relativePath = path - return res, nil -} - -func (f *SourceFile) ExtraIncludePath() *paths.Path { - return f.extraIncludePath -} - -func (f *SourceFile) SourcePath() *paths.Path { - return f.sourceRoot.JoinPath(f.relativePath) -} - -func (f *SourceFile) ObjectPath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".o") -} - -func (f *SourceFile) DepfilePath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".d") -} - type Command interface { Run(ctx *Context) error } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 102b007883f..1cddbd3bb48 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -20,42 +20,12 @@ import ( "os" "os/exec" "strings" - "unicode" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/text/runes" - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" ) -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} - -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") - -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] -} - -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { - // See if the file is readable by opening it - f, err := file.Open() - if err != nil { - return false - } - f.Close() - return true -} - -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.Replace(arg, "\\", "\\\\", -1) @@ -126,29 +96,6 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { - fileFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - paths.FilterOutDirectories(), - FilterReadableFiles, - ) - if len(extensions) > 0 { - fileFilter = paths.AndFilter( - paths.FilterSuffixes(extensions...), - fileFilter, - ) - } - if recurse { - dirFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - ) - return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) - } - return dir.ReadDir(fileFilter) -} - type loggerAction struct { onlyIfVerbose bool warn bool @@ -169,11 +116,3 @@ func (l *loggerAction) Run(ctx *types.Context) error { func LogIfVerbose(warn bool, msg string) types.Command { return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} } - -// Normalizes an UTF8 byte slice -// TODO: use it more often troughout all the project (maybe on logger interface?) -func NormalizeUTF8(buf []byte) []byte { - t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) - result, _, _ := transform.Bytes(t, buf) - return result -} From a64582c8acca1d18dde3849ca44b8df243c2807e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Mon, 4 Sep 2023 17:29:42 +0200 Subject: [PATCH 16/18] refactoring the cmd.Exec in favour of executils --- arduino/builder/compilation_database.go | 8 ++--- arduino/builder/compilation_database_test.go | 5 +-- arduino/builder/cpp/cpp.go | 4 +++ arduino/builder/preprocessor/gcc.go | 4 +-- arduino/builder/utils/utils.go | 3 -- executils/process.go | 10 ++++++ legacy/builder/builder_utils/utils.go | 26 +++++++++------ legacy/builder/create_cmake_rule.go | 4 +-- legacy/builder/phases/core_builder.go | 4 +-- legacy/builder/phases/libraries_builder.go | 12 +++---- legacy/builder/phases/linker.go | 6 ++-- legacy/builder/phases/sizer.go | 4 +-- legacy/builder/phases/sketch_builder.go | 4 +-- legacy/builder/recipe_runner.go | 4 +-- legacy/builder/utils/utils.go | 33 +++++++------------- 15 files changed, 70 insertions(+), 61 deletions(-) diff --git a/arduino/builder/compilation_database.go b/arduino/builder/compilation_database.go index 9d82cb88382..6a7e2e475dd 100644 --- a/arduino/builder/compilation_database.go +++ b/arduino/builder/compilation_database.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" "os" - "os/exec" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" ) @@ -68,8 +68,8 @@ func (db *CompilationDatabase) SaveToFile() { } // Add adds a new CompilationDatabase entry -func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { - commandDir := command.Dir +func (db *CompilationDatabase) Add(target *paths.Path, command *executils.Process) { + commandDir := command.GetDir() if commandDir == "" { // This mimics what Cmd.Run also does: Use Dir if specified, // current directory otherwise @@ -82,7 +82,7 @@ func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { entry := CompilationCommand{ Directory: commandDir, - Arguments: command.Args, + Arguments: command.GetArgs(), File: target.String(), } diff --git a/arduino/builder/compilation_database_test.go b/arduino/builder/compilation_database_test.go index c7f5bc3003d..8a715533617 100644 --- a/arduino/builder/compilation_database_test.go +++ b/arduino/builder/compilation_database_test.go @@ -16,9 +16,9 @@ package builder import ( - "os/exec" "testing" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -28,7 +28,8 @@ func TestCompilationDatabase(t *testing.T) { require.NoError(t, err) defer tmpfile.Remove() - cmd := exec.Command("gcc", "arg1", "arg2") + cmd, err := executils.NewProcess(nil, "gcc", "arg1", "arg2") + require.NoError(t, err) db := NewCompilationDatabase(tmpfile) db.Add(paths.New("test"), cmd) db.SaveToFile() diff --git a/arduino/builder/cpp/cpp.go b/arduino/builder/cpp/cpp.go index 71c2b696702..4eaa4fafcd4 100644 --- a/arduino/builder/cpp/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -106,3 +106,7 @@ func ParseString(line string) (string, string, bool) { i += width } } + +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 9f63eaa93df..2029f08b972 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -38,7 +38,7 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths. gccBuildProperties.SetPath("source_file", sourceFilePath) gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath) - includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) + includesStrings := f.Map(includes.AsStrings(), cpp.WrapWithHyphenI) gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) const gccPreprocRecipeProperty = "recipe.preproc.macros" diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index 64c1a54dbdd..b1ee7b0bac8 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -180,6 +180,3 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat return dir.ReadDir(fileFilter) } -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} diff --git a/executils/process.go b/executils/process.go index 7e0ac135089..137d2f86bc3 100644 --- a/executils/process.go +++ b/executils/process.go @@ -137,6 +137,11 @@ func (p *Process) SetDir(dir string) { p.cmd.Dir = dir } +// GetDir gets the working directory of the command. +func (p *Process) GetDir() string { + return p.cmd.Dir +} + // SetDirFromPath sets the working directory of the command. If path is nil, Run // runs the command in the calling process's current directory. func (p *Process) SetDirFromPath(path *paths.Path) { @@ -187,3 +192,8 @@ func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, erro err := p.RunWithinContext(ctx) return stdout.Bytes(), stderr.Bytes(), err } + +// GetArgs returns the command arguments +func (p *Process) GetArgs() []string { + return p.cmd.Args +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index a827e496dac..1d53860b163 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -18,7 +18,6 @@ package builder_utils import ( "fmt" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -26,6 +25,7 @@ import ( bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -173,7 +173,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - command, err := PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, recipe, false) if err != nil { return nil, errors.WithStack(err) } @@ -244,7 +244,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath) properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) - command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return nil, errors.WithStack(err) } @@ -260,7 +260,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile const COMMANDLINE_LIMIT = 30000 -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool, toolEnv []string) (*exec.Cmd, error) { +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { pattern := buildProperties.Get(recipe) if pattern == "" { return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) @@ -275,24 +275,30 @@ func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, rem if err != nil { return nil, errors.WithStack(err) } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), toolEnv...) // if the overall commandline is too long for the platform // try reducing the length by making the filenames relative // and changing working directory to build.path + var relativePath string if len(commandLine) > COMMANDLINE_LIMIT { - relativePath := buildProperties.Get("build.path") - for i, arg := range command.Args { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { if _, err := os.Stat(arg); os.IsNotExist(err) { continue } rel, err := filepath.Rel(relativePath, arg) if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - command.Args[i] = rel + parts[i] = rel } } - command.Dir = relativePath + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) } return command, nil diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 45f68ef9ed4..6d6b67ae1d6 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -367,9 +367,9 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) - for _, arg := range command.Args { + for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { *defines = appendIfNotPresent(*defines, arg) continue diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 455588f7444..0b629365eba 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -22,13 +22,13 @@ import ( "os" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -80,7 +80,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } - includes = f.Map(includes, utils.WrapWithHyphenI) + includes = f.Map(includes, cpp.WrapWithHyphenI) var err error diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 88106c9a9de..832bc9b0f8f 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,7 +18,7 @@ package phases import ( "strings" - "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -38,7 +38,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() - includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { @@ -67,9 +67,9 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -78,7 +78,7 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib } } } - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FLOAT_ABI_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -201,7 +201,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } } else { if library.UtilityDir != nil { - includes = append(includes, utils.WrapWithHyphenI(library.UtilityDir.String())) + includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } libObjectFiles, err := builder_utils.CompileFiles(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) if err != nil { diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index cc7c293199d..1f324576586 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -93,7 +93,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return errors.WithStack(err) } @@ -114,10 +114,10 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err - } +} _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) return err diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index d466aca2b06..c064fbe717b 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -50,7 +50,7 @@ func (s *Sizer) Run(ctx *types.Context) error { } func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -179,7 +179,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 398ba87de8b..51057a21835 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,7 +16,7 @@ package phases import ( - "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/cpp" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -29,7 +29,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() - includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index a6b552b2c6d..01257267287 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -44,14 +44,14 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } if ctx.OnlyUpdateCompilationDatabase && s.SkipIfOnlyUpdatingCompilationDatabase { if ctx.Verbose { - ctx.Info(tr("Skipping: %[1]s", strings.Join(command.Args, " "))) + ctx.Info(tr("Skipping: %[1]s", strings.Join(command.GetArgs(), " "))) } return nil } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 1cddbd3bb48..ec34d1f2c3b 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -18,9 +18,9 @@ package utils import ( "bytes" "os" - "os/exec" "strings" + "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" @@ -51,30 +51,30 @@ const ( Capture = 3 // Capture into buffer ) -func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { +func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) { if ctx.Verbose { - ctx.Info(PrintableCommand(command.Args)) + ctx.Info(PrintableCommand(command.GetArgs())) } + stdoutBuffer := &bytes.Buffer{} if stdout == Capture { - buffer := &bytes.Buffer{} - command.Stdout = buffer + command.RedirectStdoutTo(stdoutBuffer) } else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { if ctx.Stdout != nil { - command.Stdout = ctx.Stdout + command.RedirectStdoutTo(ctx.Stdout) } else { - command.Stdout = os.Stdout + command.RedirectStdoutTo(os.Stdout) } } + stderrBuffer := &bytes.Buffer{} if stderr == Capture { - buffer := &bytes.Buffer{} - command.Stderr = buffer + command.RedirectStderrTo(stderrBuffer) } else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { if ctx.Stderr != nil { - command.Stderr = ctx.Stderr + command.RedirectStderrTo(ctx.Stderr) } else { - command.Stderr = os.Stderr + command.RedirectStderrTo(os.Stderr) } } @@ -84,16 +84,7 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) } err = command.Wait() - - var outbytes, errbytes []byte - if buf, ok := command.Stdout.(*bytes.Buffer); ok { - outbytes = buf.Bytes() - } - if buf, ok := command.Stderr.(*bytes.Buffer); ok { - errbytes = buf.Bytes() - } - - return outbytes, errbytes, errors.WithStack(err) + return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } type loggerAction struct { From 53b84b76db6c6aed88cf0c2d9f3d0bd35e8f5523 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 09:51:22 +0200 Subject: [PATCH 17/18] use detector FindIncludes in tests --- legacy/builder/builder.go | 3 +- legacy/builder/test/builder_test.go | 5 +- legacy/builder/test/builder_utils_test.go | 16 +- .../test/includes_finder_with_regexp_test.go | 14 +- .../test/includes_to_include_folders_test.go | 106 +++++- legacy/builder/test/libraries_loader_test.go | 10 +- legacy/builder/test/prototypes_adder_test.go | 305 +++++++++++++++--- .../unused_compiled_libraries_remover_test.go | 8 +- 8 files changed, 393 insertions(+), 74 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index c9336d24528..ff517a5a2d4 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -223,7 +223,7 @@ func RunPreprocess(ctx *types.Context) error { func findIncludes(ctx *types.Context) types.BareCommand { return types.BareCommand(func(ctx *types.Context) error { - ctx.SketchLibrariesDetector.FindIncludes( + return ctx.SketchLibrariesDetector.FindIncludes( ctx.BuildPath, ctx.BuildProperties.GetPath("build.core.path"), ctx.BuildProperties.GetPath("build.variant.path"), @@ -233,6 +233,5 @@ func findIncludes(ctx *types.Context) types.BareCommand { ctx.BuildProperties, ctx.TargetPlatform.Platform.Architecture, ) - return nil }) } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 10b4f4a4a24..b8eb317f2e9 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -23,6 +23,7 @@ import ( "time" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder" @@ -127,14 +128,14 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat } if !stepToSkip[skipLibraries] { - lm, libsResolver, _, err := bldr.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) NoError(t, err) - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( lm, libsResolver, ctx.Verbose, false, diff --git a/legacy/builder/test/builder_utils_test.go b/legacy/builder/test/builder_utils_test.go index b20e7e2555a..f032eabeddc 100644 --- a/legacy/builder/test/builder_utils_test.go +++ b/legacy/builder/test/builder_utils_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -42,7 +42,7 @@ func TestObjFileIsUpToDateObjMissing(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, nil, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, nil, nil) NoError(t, err) require.False(t, upToDate) } @@ -54,7 +54,7 @@ func TestObjFileIsUpToDateDepMissing(t *testing.T) { objFile := tempFile(t, "obj") defer objFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, nil) NoError(t, err) require.False(t, upToDate) } @@ -70,7 +70,7 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -86,7 +86,7 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer depFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -110,7 +110,7 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { data := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(data)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -132,7 +132,7 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { res := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -156,7 +156,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { res := sourceFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } diff --git a/legacy/builder/test/includes_finder_with_regexp_test.go b/legacy/builder/test/includes_finder_with_regexp_test.go index 3a55415aa88..16dd07da48f 100644 --- a/legacy/builder/test/includes_finder_with_regexp_test.go +++ b/legacy/builder/test/includes_finder_with_regexp_test.go @@ -18,7 +18,7 @@ package test import ( "testing" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/stretchr/testify/require" ) @@ -27,13 +27,13 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { - include := builder.IncludesFinderWithRegExp("") + include := detector.IncludesFinderWithRegExp("") require.Equal(t, "", include) } @@ -43,7 +43,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -53,7 +53,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -62,7 +62,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } @@ -71,7 +71,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "register.h", include) } diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index ed6afc51d1e..b0c19fa9611 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -20,7 +20,6 @@ import ( "sort" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -37,7 +36,18 @@ func TestIncludesToIncludeFolders(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -60,7 +70,18 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -82,7 +103,18 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -107,7 +139,18 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -137,7 +180,18 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -168,7 +222,19 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + + }), } for _, command := range commands { err := command.Run(ctx) @@ -199,7 +265,18 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -225,7 +302,18 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index dc4707455ff..e4e1cacfa59 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -20,8 +20,8 @@ import ( "sort" "testing" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -47,7 +47,7 @@ func TestLoadLibrariesAVR(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - lm, libsResolver, _, err := builder.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -151,7 +151,7 @@ func TestLoadLibrariesSAM(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - lm, libsResolver, _, err := builder.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -228,7 +228,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - lm, _, _, err := builder.LibrariesLoader( + lm, _, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -251,7 +251,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - lm, _, _, err := builder.LibrariesLoader( + lm, _, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 56ddc806c0c..9adbd1d1a7d 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -47,7 +47,18 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -71,7 +82,18 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -81,7 +103,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithIfDef", "SketchWithIfDef.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderBaladuino(t *testing.T) { @@ -95,7 +117,18 @@ func TestPrototypesAdderBaladuino(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -105,7 +138,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("Baladuino", "Baladuino.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { @@ -119,7 +152,18 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -129,7 +173,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { @@ -143,7 +187,18 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -153,7 +208,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderLineContinuations(t *testing.T) { @@ -167,7 +222,18 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -177,7 +243,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("LineContinuations", "LineContinuations.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderStringWithComment(t *testing.T) { @@ -191,7 +257,18 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -201,7 +278,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("StringWithComment", "StringWithComment.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderSketchWithStruct(t *testing.T) { @@ -215,7 +292,18 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -225,7 +313,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithStruct", "SketchWithStruct.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) - obtained := strings.Replace(preprocessedSketch, "\r\n", "\n", -1) + obtained := strings.ReplaceAll(preprocessedSketch, "\r\n", "\n") // ctags based preprocessing removes the space after "dostuff", but this is still OK // TODO: remove this exception when moving to a more powerful parser preprocessed = strings.Replace(preprocessed, "void dostuff (A_NEW_TYPE * bar);", "void dostuff(A_NEW_TYPE * bar);", 1) @@ -247,7 +335,18 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -260,7 +359,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { require.Contains(t, preprocessedSketch, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n") preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { @@ -277,7 +376,18 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -304,7 +414,18 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -332,7 +453,18 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -359,7 +491,18 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -374,12 +517,12 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { obtained := preprocessedSketch // ctags based preprocessing removes "inline" but this is still OK // TODO: remove this exception when moving to a more powerful parser - expected = strings.Replace(expected, "static inline int8_t testInline();", "static int8_t testInline();", -1) - obtained = strings.Replace(obtained, "static inline int8_t testInline();", "static int8_t testInline();", -1) + expected = strings.ReplaceAll(expected, "static inline int8_t testInline();", "static int8_t testInline();") + obtained = strings.ReplaceAll(obtained, "static inline int8_t testInline();", "static int8_t testInline();") // ctags based preprocessing removes "__attribute__ ....." but this is still OK // TODO: remove this exception when moving to a more powerful parser - expected = strings.Replace(expected, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1) - obtained = strings.Replace(obtained, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1) + expected = strings.ReplaceAll(expected, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();") + obtained = strings.ReplaceAll(obtained, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();") require.Contains(t, obtained, expected) } @@ -397,7 +540,18 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -430,7 +584,18 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -462,7 +627,18 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -476,8 +652,8 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { obtained := preprocessedSketch // ctags based preprocessing ignores line with typename // TODO: remove this exception when moving to a more powerful parser - expected = strings.Replace(expected, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "", -1) - obtained = strings.Replace(obtained, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "", -1) + expected = strings.ReplaceAll(expected, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "") + obtained = strings.ReplaceAll(obtained, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "") require.Contains(t, obtained, expected) } @@ -495,7 +671,18 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -508,7 +695,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { require.Contains(t, preprocessedSketch, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { @@ -525,7 +712,18 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -538,7 +736,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { require.Contains(t, preprocessedSketch, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.ReplaceAll(preprocessedSketch, "\r\n", "\n")) } func TestPrototypesAdderSketchWithConst(t *testing.T) { @@ -555,7 +753,18 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -579,7 +788,18 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) @@ -603,7 +823,18 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, + types.BareCommand(func(ctx *types.Context) error { + return ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + }), } for _, command := range commands { err := command.Run(ctx) diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index ab49dc9e26f..24001e674ab 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -18,7 +18,7 @@ package test import ( "testing" - bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -37,7 +37,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -60,7 +60,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -80,7 +80,7 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { NoError(t, temp.Join("dummy_file").WriteFile([]byte{})) ctx := &types.Context{} - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.LibrariesBuildPath = temp From 066c153e6c848c2d2ad41b4f753d4e1b6edff76e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 09:59:20 +0200 Subject: [PATCH 18/18] add comments and make some plubic methods private --- arduino/builder/cpp/cpp.go | 1 + arduino/builder/detector/detector.go | 103 ++++++++++++++------------- arduino/builder/utils/utils.go | 23 +++--- 3 files changed, 64 insertions(+), 63 deletions(-) diff --git a/arduino/builder/cpp/cpp.go b/arduino/builder/cpp/cpp.go index 4eaa4fafcd4..53123434fe1 100644 --- a/arduino/builder/cpp/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -107,6 +107,7 @@ func ParseString(line string) (string, string, bool) { } } +// WrapWithHyphenI fixdoc func WrapWithHyphenI(value string) string { return "\"-I" + value + "\"" } diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go index aa00a02e718..dc64b0319e5 100644 --- a/arduino/builder/detector/detector.go +++ b/arduino/builder/detector/detector.go @@ -146,11 +146,6 @@ func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Lib l.importedLibraries = append(l.importedLibraries, library) } -// UseCachedLibrariesResolution todo -func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { - return l.useCachedLibrariesResolution -} - // PrintUsedAndNotUsedLibraries todo func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { // Print this message: @@ -182,12 +177,13 @@ func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) time.Sleep(100 * time.Millisecond) } +// IncludeFolders fixdoc func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { // TODO should we do a deep copy? return l.includeFolders } -// AppendIncludeFolder todo should rename this, probably after refactoring the +// appendIncludeFolder todo should rename this, probably after refactoring the // container_find_includes command. // Original comment: // Append the given folder to the include path and match or append it to @@ -195,7 +191,7 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { // include (e.g. what #include line in what file it was resolved from) // and should be the empty string for the default include folders, like // the core or variant. -func (l *SketchLibrariesDetector) AppendIncludeFolder( +func (l *SketchLibrariesDetector) appendIncludeFolder( cache *includeCache, sourceFilePath *paths.Path, include string, @@ -242,13 +238,13 @@ func (l *SketchLibrariesDetector) findIncludes( ) error { librariesResolutionCache := buildPath.Join("libraries.cache") if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { + d, err := librariesResolutionCache.ReadFile() + if err != nil { + return err + } + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { return err - } else { - includeFolders := l.includeFolders - if err := json.Unmarshal(d, &includeFolders); err != nil { - return err - } } if l.verbose { l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) @@ -259,20 +255,20 @@ func (l *SketchLibrariesDetector) findIncludes( cachePath := buildPath.Join("includes.cache") cache := readCache(cachePath) - l.AppendIncludeFolder(cache, nil, "", buildCorePath) + l.appendIncludeFolder(cache, nil, "", buildCorePath) if buildVariantPath != nil { - l.AppendIncludeFolder(cache, nil, "", buildVariantPath) + l.appendIncludeFolder(cache, nil, "", buildVariantPath) } - sourceFileQueue := &UniqueSourceFileQueue{} + sourceFileQueue := &uniqueSourceFileQueue{} if !l.useCachedLibrariesResolution { sketch := sketch - mergedfile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + mergedfile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { return errors.WithStack(err) } - sourceFileQueue.Push(mergedfile) + sourceFileQueue.push(mergedfile) l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) srcSubfolderPath := sketchBuildPath.Join("src") @@ -280,7 +276,7 @@ func (l *SketchLibrariesDetector) findIncludes( l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) } - for !sourceFileQueue.Empty() { + for !sourceFileQueue.empty() { err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) if err != nil { cachePath.Remove() @@ -310,13 +306,13 @@ func (l *SketchLibrariesDetector) findIncludes( func (l *SketchLibrariesDetector) findIncludesUntilDone( cache *includeCache, - sourceFileQueue *UniqueSourceFileQueue, + sourceFileQueue *uniqueSourceFileQueue, buildProperties *properties.Map, sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, platformArch string, ) error { - sourceFile := sourceFileQueue.Pop() + sourceFile := sourceFileQueue.pop() sourcePath := sourceFile.SourcePath() targetFilePath := paths.NullPath() depPath := sourceFile.DepfilePath() @@ -414,7 +410,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( // include path and queue its source files for further // include scanning l.AppendImportedLibraries(library) - l.AppendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + l.appendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) if library.Precompiled && library.PrecompiledWithSources { // Fully precompiled libraries should have no dependencies to avoid ABI breakage @@ -433,7 +429,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, - sourceFileQueue *UniqueSourceFileQueue, + sourceFileQueue *uniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool, @@ -448,11 +444,11 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( } for _, filePath := range filePaths { - sourceFile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + sourceFile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) if err != nil { return errors.WithStack(err) } - sourceFileQueue.Push(sourceFile) + sourceFileQueue.push(sourceFile) } return nil @@ -484,10 +480,12 @@ func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { return nil } -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") +// includeRegexp fixdoc +var includeRegexp = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") +// IncludesFinderWithRegExp fixdoc func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) + match := includeRegexp.FindStringSubmatch(source) if match != nil { return strings.TrimSpace(match[1]) } @@ -507,7 +505,7 @@ func findIncludeForOldCompilers(source string) string { return "" } -type SourceFile struct { +type sourceFile struct { // Path to the source file within the sketch/library root folder relativePath *paths.Path @@ -528,22 +526,23 @@ type SourceFile struct { buildRoot *paths.Path } -func (f *SourceFile) Equals(g *SourceFile) bool { +// Equals fixdoc +func (f *sourceFile) Equals(g *sourceFile) bool { return f.relativePath.EqualsTo(g.relativePath) && f.buildRoot.EqualsTo(g.buildRoot) && f.sourceRoot.EqualsTo(g.sourceRoot) } -// Create a SourceFile containing the given source file path within the +// makeSourceFile containing the given source file path within the // given origin. The given path can be absolute, or relative within the // origin's root source folder -func MakeSourceFile( +func makeSourceFile( sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, origin interface{}, path *paths.Path, -) (*SourceFile, error) { - res := &SourceFile{} +) (*sourceFile, error) { + res := &sourceFile{} switch o := origin.(type) { case *sketch.Sketch: @@ -568,19 +567,23 @@ func MakeSourceFile( return res, nil } -func (f *SourceFile) ExtraIncludePath() *paths.Path { +// ExtraIncludePath fixdoc +func (f *sourceFile) ExtraIncludePath() *paths.Path { return f.extraIncludePath } -func (f *SourceFile) SourcePath() *paths.Path { +// SourcePath fixdoc +func (f *sourceFile) SourcePath() *paths.Path { return f.sourceRoot.JoinPath(f.relativePath) } -func (f *SourceFile) ObjectPath() *paths.Path { +// ObjectPath fixdoc +func (f *sourceFile) ObjectPath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".o") } -func (f *SourceFile) DepfilePath() *paths.Path { +// DepfilePath fixdoc +func (f *sourceFile) DepfilePath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".d") } @@ -658,21 +661,19 @@ func LibrariesLoader( return lm, resolver, verboseOut.Bytes(), nil } -func (l *SketchLibrariesDetector) OnlyUpdateCompilationDatabase() bool { - return l.onlyUpdateCompilationDatabase -} - type includeCacheEntry struct { Sourcefile *paths.Path Include string Includepath *paths.Path } +// String fixdoc func (entry *includeCacheEntry) String() string { return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", entry.Sourcefile, entry.Include, entry.Includepath) } +// Equals fixdoc func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { return entry.String() == other.String() } @@ -686,14 +687,14 @@ type includeCache struct { entries []*includeCacheEntry } -// Return the next cache entry. Should only be called when the cache is +// Next Return the next cache entry. Should only be called when the cache is // valid and a next entry is available (the latter can be checked with // ExpectFile). Does not advance the cache. func (cache *includeCache) Next() *includeCacheEntry { return cache.entries[cache.next] } -// Check that the next cache entry is about the given file. If it is +// ExpectFile check that the next cache entry is about the given file. If it is // not, or no entry is available, the cache is invalidated. Does not // advance the cache. func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { @@ -703,7 +704,7 @@ func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { } } -// Check that the next entry matches the given values. If so, advance +// ExpectEntry check that the next entry matches the given values. If so, advance // the cache. If not, the cache is invalidated. If the cache is // invalidated, or was already invalid, an entry with the given values // is appended. @@ -723,7 +724,7 @@ func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, l } } -// Check that the cache is completely consumed. If not, the cache is +// ExpectEnd check that the cache is completely consumed. If not, the cache is // invalidated. func (cache *includeCache) ExpectEnd() { if cache.valid && cache.next < len(cache.entries) { @@ -771,25 +772,25 @@ func writeCache(cache *includeCache, path *paths.Path) error { return nil } -type UniqueSourceFileQueue []*SourceFile +type uniqueSourceFileQueue []*sourceFile -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { +func (queue *uniqueSourceFileQueue) push(value *sourceFile) { + if !queue.contains(value) { *queue = append(*queue, value) } } -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { +func (queue uniqueSourceFileQueue) contains(target *sourceFile) bool { return slices.ContainsFunc(queue, target.Equals) } -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { +func (queue *uniqueSourceFileQueue) pop() *sourceFile { old := *queue x := old[0] *queue = old[1:] return x } -func (queue UniqueSourceFileQueue) Empty() bool { +func (queue uniqueSourceFileQueue) empty() bool { return len(queue) == 0 } diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index b1ee7b0bac8..ae64dacb47c 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -14,6 +14,7 @@ import ( "golang.org/x/text/unicode/norm" ) +// ObjFileIsUpToDate fixdoc func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) if objectFile == nil || dependencyFile == nil { @@ -33,9 +34,8 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool if os.IsNotExist(err) { logrus.Debugf("Not found: %v", objectFile) return false, nil - } else { - return false, errors.WithStack(err) } + return false, errors.WithStack(err) } dependencyFile = dependencyFile.Clean() @@ -44,9 +44,8 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool if os.IsNotExist(err) { logrus.Debugf("Not found: %v", dependencyFile) return false, nil - } else { - return false, errors.WithStack(err) } + return false, errors.WithStack(err) } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { @@ -128,7 +127,7 @@ func unescapeDep(s string) string { return s } -// Normalizes an UTF8 byte slice +// NormalizeUTF8 byte slice // TODO: use it more often troughout all the project (maybe on logger interface?) func NormalizeUTF8(buf []byte) []byte { t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) @@ -136,11 +135,11 @@ func NormalizeUTF8(buf []byte) []byte { return result } -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} +var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} // FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] + return !sourceControlFolders[file.Base()] } // FilterReadableFiles is a ReadDirFilter that accepts only readable files @@ -154,12 +153,13 @@ func FilterReadableFiles(file *paths.Path) bool { return true } -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") +// filterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var filterOutHiddenFiles = paths.FilterOutPrefixes(".") +// FindFilesInFolder fixdoc func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( - FilterOutHiddenFiles, + filterOutHiddenFiles, FilterOutSCCS, paths.FilterOutDirectories(), FilterReadableFiles, @@ -172,11 +172,10 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat } if recurse { dirFilter := paths.AndFilter( - FilterOutHiddenFiles, + filterOutHiddenFiles, FilterOutSCCS, ) return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) } return dir.ReadDir(fileFilter) } -