Move Preprocess and Build in arduino/builder

arduino/builder/builder.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/arduino/arduino-cli/arduino/builder/compilation"
+	"github.com/arduino/arduino-cli/arduino/builder/detector"
 	"github.com/arduino/arduino-cli/arduino/builder/logger"
 	"github.com/arduino/arduino-cli/arduino/builder/progress"
 	"github.com/arduino/arduino-cli/arduino/cores"
@@ -223,14 +224,243 @@ func (b *Builder) ExecutableSectionsSize() ExecutablesFileSections {
 	return b.executableSectionsSize
 }
 
-// SaveCompilationDatabase fixdoc
-func (b *Builder) SaveCompilationDatabase() {
-	if b.compilationDatabase != nil {
-		b.compilationDatabase.SaveToFile()
-	}
-}
-
 // TargetPlatform fixdoc
 func (b *Builder) TargetPlatform() *cores.PlatformRelease {
 	return b.targetPlatform
 }
+
+// Preprocess fixdoc
+func (b *Builder) Preprocess(detector *detector.SketchLibrariesDetector) error {
+	b.Progress.AddSubSteps(6)
+	defer b.Progress.RemoveSubSteps()
+	return b.preprocess(detector)
+}
+
+func (b *Builder) preprocess(detector *detector.SketchLibrariesDetector) error {
+	if err := b.buildPath.MkdirAll(); err != nil {
+		return err
+	}
+
+	if err := b.BuildOptionsManager.WipeBuildPath(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.prebuild", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.PrepareSketchBuildPath(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.logIfVerbose(false, tr("Detecting libraries used..."))
+	err := detector.FindIncludes(
+		b.GetBuildPath(),
+		b.GetBuildProperties().GetPath("build.core.path"),
+		b.GetBuildProperties().GetPath("build.variant.path"),
+		b.GetSketchBuildPath(),
+		b.Sketch(),
+		b.GetLibrariesBuildPath(),
+		b.GetBuildProperties(),
+		b.TargetPlatform().Platform.Architecture,
+	)
+	if err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.WarnAboutArchIncompatibleLibraries(detector.ImportedLibraries())
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.logIfVerbose(false, tr("Generating function prototypes..."))
+	if err := b.PreprocessSketch(detector.IncludeFolders()); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	// Output arduino-preprocessed source
+	preprocessedSketch, err := b.sketchBuildPath.Join(b.sketch.MainFile.Base() + ".cpp").ReadFile()
+	if err != nil {
+		return err
+	}
+	b.logger.WriteStdout(preprocessedSketch)
+
+	return nil
+}
+
+func (b *Builder) logIfVerbose(warn bool, msg string) {
+	if !b.logger.Verbose() {
+		return
+	}
+	if warn {
+		b.logger.Warn(msg)
+		return
+	}
+	b.logger.Info(msg)
+}
+
+// Build fixdoc
+func (b *Builder) Build(detector *detector.SketchLibrariesDetector) error {
+	b.Progress.AddSubSteps(6 /** preprocess **/ + 21 /** build **/)
+	defer b.Progress.RemoveSubSteps()
+
+	if err := b.preprocess(detector); err != nil {
+		return err
+	}
+
+	buildErr := b.build(detector)
+
+	detector.PrintUsedAndNotUsedLibraries(buildErr != nil)
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.PrintUsedLibraries(detector.ImportedLibraries())
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if buildErr != nil {
+		return buildErr
+	}
+	if err := b.ExportProjectCMake(detector.ImportedLibraries(), detector.IncludeFolders()); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.Size(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	return nil
+}
+
+// Build fixdoc
+func (b *Builder) build(detector *detector.SketchLibrariesDetector) error {
+	b.logIfVerbose(false, tr("Compiling sketch..."))
+	if err := b.RunRecipe("recipe.hooks.sketch.prebuild", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.BuildSketch(detector.IncludeFolders()); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.sketch.postbuild", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.logIfVerbose(false, tr("Compiling libraries..."))
+	if err := b.RunRecipe("recipe.hooks.libraries.prebuild", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RemoveUnusedCompiledLibraries(detector.ImportedLibraries()); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.BuildLibraries(detector.IncludeFolders(), detector.ImportedLibraries()); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.libraries.postbuild", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.logIfVerbose(false, tr("Compiling core..."))
+	if err := b.RunRecipe("recipe.hooks.core.prebuild", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.BuildCore(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.core.postbuild", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	b.logIfVerbose(false, tr("Linking everything together..."))
+	if err := b.RunRecipe("recipe.hooks.linking.prelink", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.Link(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.linking.postlink", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.objcopy.preobjcopy", ".pattern", false); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.objcopy.", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.objcopy.postobjcopy", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.MergeSketchWithBootloader(); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if err := b.RunRecipe("recipe.hooks.postbuild", ".pattern", true); err != nil {
+		return err
+	}
+	b.Progress.CompleteStep()
+	b.Progress.PushProgress()
+
+	if b.compilationDatabase != nil {
+		b.compilationDatabase.SaveToFile()
+	}
+	return nil
+}

arduino/builder/export_cmake.go

@@ -35,11 +35,7 @@ import (
 var lineMatcher = regexp.MustCompile(`^#line\s\d+\s"`)
 
 // ExportProjectCMake fixdoc
-func (b *Builder) ExportProjectCMake(
-	sketchError bool, // Was there an error while compiling the sketch?
-	importedLibraries libraries.List,
-	includeFolders paths.PathList,
-) error {
+func (b *Builder) ExportProjectCMake(importedLibraries libraries.List, includeFolders paths.PathList) error {
 	// copies the contents of the file named src to the file named
 	// by dst. The file will be created if it does not already exist. If the
 	// destination file exists, all it's contents will be replaced by the contents
@@ -175,8 +171,8 @@ func (b *Builder) ExportProjectCMake(
 	}
 	var validStaticLibExtensions = []string{".a"}
 
-	//	If sketch error or cannot export Cmake project
-	if sketchError || b.buildProperties.Get("compiler.export_cmake") == "" {
+	//	If cannot export Cmake project
+	if b.buildProperties.Get("compiler.export_cmake") == "" {
 		return nil
 	}
 

arduino/builder/sizer.go

@@ -51,8 +51,8 @@ func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.Execut
 }
 
 // Size fixdoc
-func (b *Builder) Size(sketchError bool) error {
-	if b.onlyUpdateCompilationDatabase || sketchError {
+func (b *Builder) Size() error {
+	if b.onlyUpdateCompilationDatabase {
 		return nil
 	}
 

commands/compile/compile.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino"
-	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/builder/detector"
 	"github.com/arduino/arduino-cli/arduino/builder/logger"
 	"github.com/arduino/arduino-cli/arduino/builder/progress"
@@ -37,8 +37,6 @@ import (
 	"github.com/arduino/arduino-cli/configuration"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/internal/inventory"
-	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/sirupsen/logrus"
@@ -174,16 +172,14 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		return nil, err
 	}
 
-	builderCtx := &types.Context{}
 	actualPlatform := buildPlatform
 	builtinLibrariesDir := configuration.IDEBuiltinLibrariesDir(configuration.Settings)
 	otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
 	otherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings))
 
 	builderLogger := logger.New(outStream, errStream, req.GetVerbose(), req.GetWarnings())
-	builderCtx.BuilderLogger = builderLogger
 
-	sketchBuilder, err := bldr.NewBuilder(
+	sketchBuilder, err := builder.NewBuilder(
 		sk,
 		boardBuildProperties,
 		buildPath,
@@ -207,14 +203,13 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		if strings.Contains(err.Error(), "invalid build properties") {
 			return nil, &arduino.InvalidArgumentError{Message: tr("Invalid build properties"), Cause: err}
 		}
-		if errors.Is(err, bldr.ErrSketchCannotBeLocatedInBuildPath) {
+		if errors.Is(err, builder.ErrSketchCannotBeLocatedInBuildPath) {
 			return r, &arduino.CompileFailedError{
 				Message: tr("Sketch cannot be located in build path. Please specify a different build path"),
 			}
 		}
 		return r, &arduino.CompileFailedError{Message: err.Error()}
 	}
-	builderCtx.Builder = sketchBuilder
 
 	var libsManager *librariesmanager.LibrariesManager
 	if pme.GetProfile() != nil {
@@ -235,7 +230,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		builderLogger.Warn(string(verboseOut))
 	}
 
-	builderCtx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector(
+	sketchLibrariesDetector := detector.NewSketchLibrariesDetector(
 		libsManager, libsResolver,
 		useCachedLibrariesResolution,
 		req.GetCreateCompilationDatabaseOnly(),
@@ -270,7 +265,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	if req.GetPreprocess() {
 		// Just output preprocessed source code and exit
-		compileErr := builder.RunPreprocess(builderCtx)
+		compileErr := sketchBuilder.Preprocess(sketchLibrariesDetector)
 		if compileErr != nil {
 			compileErr = &arduino.CompileFailedError{Message: compileErr.Error()}
 		}
@@ -279,7 +274,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	defer func() {
 		importedLibs := []*rpc.Library{}
-		for _, lib := range builderCtx.SketchLibrariesDetector.ImportedLibraries() {
+		for _, lib := range sketchLibrariesDetector.ImportedLibraries() {
 			rpcLib, err := lib.ToRPCLibrary()
 			if err != nil {
 				msg := tr("Error getting information for library %s", lib.Name) + ": " + err.Error() + "\n"
@@ -315,7 +310,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 				targetBoard.String(), "'build.board'", sketchBuilder.GetBuildProperties().Get("build.board")) + "\n"))
 	}
 
-	if err := builder.RunBuilder(builderCtx); err != nil {
+	if err := sketchBuilder.Build(sketchLibrariesDetector); err != nil {
 		return r, &arduino.CompileFailedError{Message: err.Error()}
 	}
 

internal/integrationtest/daemon/daemon_test.go

@@ -187,7 +187,7 @@ func TestDaemonCompileOptions(t *testing.T) {
 	// https://github.com/arduino/arduino-cli/issues/2016
 	// assert that the task progress is increasing and doesn't contain multiple 100% values
 	results := analyzer.Results[""]
-	require.True(t, results[len(results)-1].Completed)
+	require.True(t, results[len(results)-1].Completed, fmt.Sprintf("latest percent value: %v", results[len(results)-1].Percent))
 	require.IsNonDecreasing(t, f.Map(results, (*commands.TaskProgress).GetPercent))
 }