Using build-cache path to store libs cache / added cache expire every day

Author: cmaglie

commands/instances.go

@@ -92,7 +92,7 @@ func (s *arduinoCoreServerImpl) Create(ctx context.Context, req *rpc.CreateReque
 	if err != nil {
 		return nil, err
 	}
-	inst, err := instances.Create(dataDir, packagesDir, userPackagesDir, downloadsDir, userAgent, config)
+	inst, err := instances.Create(dataDir, packagesDir, userPackagesDir, downloadsDir, s.settings.GetBuildCachePath(), userAgent, config)
 	if err != nil {
 		return nil, err
 	}
@@ -409,7 +409,7 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 		}
 	}
 
-	lm, libsLoadingWarnings := lmb.Build()
+	lm, libsLoadingWarnings := lmb.Build(s.settings.GetBuildCachePath())
 	_ = instances.SetLibraryManager(instance, lm) // should never fail
 	for _, status := range libsLoadingWarnings {
 		logrus.WithError(status.Err()).Warnf("Error loading library")

commands/internal/instances/instances.go

@@ -134,7 +134,7 @@ func SetLibraryManager(inst *rpc.Instance, lm *librariesmanager.LibrariesManager
 }
 
 // Create a new *rpc.Instance ready to be initialized
-func Create(dataDir, packagesDir, userPackagesDir, downloadsDir *paths.Path, extraUserAgent string, downloaderConfig downloader.Config) (*rpc.Instance, error) {
+func Create(dataDir, packagesDir, userPackagesDir, downloadsDir, cacheDir *paths.Path, extraUserAgent string, downloaderConfig downloader.Config) (*rpc.Instance, error) {
 	// Create package manager
 	userAgent := "arduino-cli/" + version.VersionInfo.VersionString
 	if extraUserAgent != "" {
@@ -143,7 +143,7 @@ func Create(dataDir, packagesDir, userPackagesDir, downloadsDir *paths.Path, ext
 	tempDir := dataDir.Join("tmp")
 
 	pm := packagemanager.NewBuilder(dataDir, packagesDir, userPackagesDir, downloadsDir, tempDir, userAgent, downloaderConfig).Build()
-	lm, _ := librariesmanager.NewBuilder().Build()
+	lm, _ := librariesmanager.NewBuilder().Build(cacheDir)
 
 	instance := &coreInstance{
 		pm: pm,

internal/arduino/builder/internal/detector/detector.go

@@ -605,7 +605,7 @@ func LibrariesLoader(
 	if useCachedLibrariesResolution {
 		// Since we are using the cached libraries resolution
 		// the library manager is not needed.
-		lm, _ = librariesmanager.NewBuilder().Build()
+		lm, _ = librariesmanager.NewBuilder().Build(nil)
 	}
 	if librariesManager == nil {
 		lmb := librariesmanager.NewBuilder()
@@ -653,7 +653,7 @@ func LibrariesLoader(
 			})
 		}
 
-		newLm, libsLoadingWarnings := lmb.Build()
+		newLm, libsLoadingWarnings := lmb.Build(nil)
 		for _, status := range libsLoadingWarnings {
 			// With the refactoring of the initialization step of the CLI we changed how
 			// errors are returned when loading platforms and libraries, that meant returning a list of

internal/arduino/libraries/librariesmanager/install.go

@@ -65,7 +65,7 @@ func (lmi *Installer) InstallPrerequisiteCheck(name string, version *semver.Vers
 		return nil, err
 	}
 
-	lmi.RescanLibraries()
+	lmi.RescanLibraries(nil) // TODO: Is this required??
 
 	libs := lmi.FindByReference(name, nil, installLocation)
 	if len(libs) > 1 {

internal/arduino/libraries/librariesmanager/librariesmanager.go

@@ -16,12 +16,15 @@
 package librariesmanager
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
 	"slices"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/libraries"
@@ -124,12 +127,12 @@ func (lm *LibrariesManager) NewInstaller() (*Installer, func()) {
 }
 
 // Build builds a new LibrariesManager.
-func (lmb *Builder) Build() (*LibrariesManager, []*status.Status) {
+func (lmb *Builder) Build(cacheDir *paths.Path) (*LibrariesManager, []*status.Status) {
 	var statuses []*status.Status
 	res := &LibrariesManager{}
 	for _, dir := range lmb.librariesDir {
 		if !dir.scanned {
-			if errs := lmb.loadLibrariesFromDir(dir); len(errs) > 0 {
+			if errs := lmb.loadLibrariesFromDir(dir, cacheDir); len(errs) > 0 {
 				statuses = append(statuses, errs...)
 			}
 		}
@@ -167,11 +170,11 @@ func (lmb *Builder) AddLibrariesDir(libDir LibrariesDir) {
 }
 
 // RescanLibraries reload all installed libraries in the system.
-func (lmi *Installer) RescanLibraries() []*status.Status {
+func (lmi *Installer) RescanLibraries(cacheDir *paths.Path) []*status.Status {
 	lmi.libraries = map[string]libraries.List{}
 	statuses := []*status.Status{}
 	for _, dir := range lmi.librariesDir {
-		if errs := lmi.loadLibrariesFromDir(dir); len(errs) > 0 {
+		if errs := lmi.loadLibrariesFromDir(dir, cacheDir); len(errs) > 0 {
 			statuses = append(statuses, errs...)
 		}
 	}
@@ -194,15 +197,30 @@ func (lm *LibrariesManager) getLibrariesDir(installLocation libraries.LibraryLoc
 	}
 }
 
+func (lm *LibrariesManager) libCacheFile(librariesDir *LibrariesDir, cacheDir *paths.Path) *paths.Path {
+	if cacheDir == nil {
+		return nil
+	}
+	hash := md5.Sum([]byte(librariesDir.Path.String()))
+	cache := cacheDir.Join("libs", "cache_"+hex.EncodeToString(hash[:]))
+	if stat, err := cache.Stat(); err != nil {
+		return cache
+	} else if time.Since(stat.ModTime()) < time.Hour*24 {
+		return cache
+	}
+	cache.Remove()
+	return cache
+}
+
 // loadLibrariesFromDir loads all libraries in the given directory. Returns
 // nil if the directory doesn't exists.
-func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*status.Status {
+func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir, cacheDir *paths.Path) []*status.Status {
 	statuses := []*status.Status{}
 
 	librariesDir.scanned = true
 
 	var loadedLibs libraries.List
-	if cacheFilePath := librariesDir.Path.Join("libraries-loader-cache"); cacheFilePath.Exist() {
+	if cacheFilePath := lm.libCacheFile(librariesDir, cacheDir); cacheFilePath != nil && cacheFilePath.Exist() {
 		logrus.WithField("file", cacheFilePath).Info("Using library cache")
 
 		// Load lib cache
@@ -214,6 +232,7 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*
 		defer cache.Close()
 
 		if err := loadedLibs.UnmarshalBinary(cache); err != nil {
+			cacheFilePath.Remove()
 			s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err)
 			return append(statuses, s)
 		}
@@ -246,17 +265,20 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*
 		}
 
 		// Write lib cache
-		cache, err := cacheFilePath.Create()
-		if err != nil {
-			s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err)
-			return append(statuses, s)
-		}
-		err = loadedLibs.MarshalBinary(cache)
-		cache.Close()
-		if err != nil {
-			cacheFilePath.Remove()
-			s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err)
-			return append(statuses, s)
+		if cacheFilePath != nil {
+			cacheFilePath.Parent().MkdirAll()
+			cache, err := cacheFilePath.Create()
+			if err != nil {
+				s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err)
+				return append(statuses, s)
+			}
+			err = loadedLibs.MarshalBinary(cache)
+			cache.Close()
+			if err != nil {
+				cacheFilePath.Remove()
+				s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err)
+				return append(statuses, s)
+			}
 		}
 	}
 

internal/arduino/libraries/librariesmanager/librariesmanager_test.go

@@ -25,19 +25,19 @@ func TestLibrariesBuilderScanCloneRescan(t *testing.T) {
 	lmb := NewBuilder()
 	lmb.libraries["testLibA"] = libraries.List{}
 	lmb.libraries["testLibB"] = libraries.List{}
-	lm, warns := lmb.Build()
+	lm, warns := lmb.Build(nil)
 	require.Empty(t, warns)
 	require.Len(t, lm.libraries, 2)
 
 	// Cloning should keep existing libraries
-	lm2, warns2 := lm.Clone().Build()
+	lm2, warns2 := lm.Clone().Build(nil)
 	require.Empty(t, warns2)
 	require.Len(t, lm2.libraries, 2)
 
 	// Full rescan should update libs
 	{
 		lmi2, release := lm2.NewInstaller()
-		lmi2.RescanLibraries()
+		lmi2.RescanLibraries(nil)
 		release()
 	}
 	require.Len(t, lm.libraries, 2) // Ensure deep-coping worked as expected...