Skip to content

Commit

Permalink
Cache library scan results
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmaglie
cmaglie committed Sep 12, 2024
1 parent 23d5036 commit cbe2a4d
Show file tree
Hide file tree
Showing 2 changed files with 300 additions and 17 deletions.
231 changes: 231 additions & 0 deletions internal/arduino/libraries/libraries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,11 @@
package libraries

import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"

"github.com/arduino/arduino-cli/internal/arduino/cores"
"github.com/arduino/arduino-cli/internal/arduino/globals"
Expand Down Expand Up @@ -90,6 +93,234 @@ func (library *Library) String() string {
return library.Name + "@" + library.Version.String()
}

func (library *Library) MarshalBinary() []byte {
buffer := bytes.NewBuffer(make([]byte, 0, 4096))
writeString := func(in string) {
inBytes := []byte(in)
binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes)))
buffer.Write(inBytes)
}
writeStringArray := func(in []string) {
binary.Write(buffer, binary.NativeEndian, uint16(len(in)))
for _, i := range in {
writeString(i)
}
}
writeMap := func(in map[string]bool) {
binary.Write(buffer, binary.NativeEndian, uint16(len(in)))
for k, v := range in {
writeString(k)
binary.Write(buffer, binary.NativeEndian, v)
}
}
writePath := func(in *paths.Path) {
if in == nil {
writeString("")
} else {
writeString(in.String())
}
}
writeString(library.Name)
writeString(library.Author)
writeString(library.Maintainer)
writeString(library.Sentence)
writeString(library.Paragraph)
writeString(library.Website)
writeString(library.Category)
writeStringArray(library.Architectures)
writeStringArray(library.Types)
writePath(library.InstallDir)
writeString(library.DirName)
writePath(library.SourceDir)
writePath(library.UtilityDir)
binary.Write(buffer, binary.NativeEndian, int32(library.Location))
// library.ContainerPlatform *cores.PlatformRelease `json:""`
binary.Write(buffer, binary.NativeEndian, int32(library.Layout))
binary.Write(buffer, binary.NativeEndian, library.DotALinkage)
binary.Write(buffer, binary.NativeEndian, library.Precompiled)
binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources)
writeString(library.LDflags)
binary.Write(buffer, binary.NativeEndian, library.IsLegacy)
binary.Write(buffer, binary.NativeEndian, library.InDevelopment)
writeString(library.Version.String())
writeString(library.License)
//writeStringArray(library.Properties.AsSlice())
writeStringArray(library.Examples.AsStrings())
writeStringArray(library.declaredHeaders)
writeStringArray(library.sourceHeaders)
writeMap(library.CompatibleWith)
return buffer.Bytes()
}

func (library *Library) UnmarshalBinary(in io.Reader) error {
readString := func() (string, error) {
var len uint16

Check failure on line 157 in internal/arduino/libraries/libraries.go

View workflow job for this annotation

GitHub Actions / check-style (./) 

redefines-builtin-id: redefinition of the built-in function len (revive)
if err := binary.Read(in, binary.NativeEndian, &len); err != nil {
return "", err
}
res := make([]byte, len)
if _, err := in.Read(res); err != nil {
return "", err
}
return string(res), nil
}
readStringArray := func() ([]string, error) {
var len uint16

Check failure on line 168 in internal/arduino/libraries/libraries.go

View workflow job for this annotation

GitHub Actions / check-style (./) 

redefines-builtin-id: redefinition of the built-in function len (revive)
if err := binary.Read(in, binary.NativeEndian, &len); err != nil {
return nil, err
}
res := make([]string, len)
for i := range res {
var err error
res[i], err = readString()
if err != nil {
return nil, err
}
}
return res, nil
}
readMap := func() (map[string]bool, error) {
var len uint16

Check failure on line 183 in internal/arduino/libraries/libraries.go

View workflow job for this annotation

GitHub Actions / check-style (./) 

redefines-builtin-id: redefinition of the built-in function len (revive)
if err := binary.Read(in, binary.NativeEndian, &len); err != nil {
return nil, err
}
res := map[string]bool{}
for i := uint16(0); i < len; i++ {
k, err := readString()
if err != nil {
return nil, err
}
var v bool
if err := binary.Read(in, binary.NativeEndian, &v); err != nil {
return nil, err
}
res[k] = v
}
return res, nil
}
var err error
library.Name, err = readString()
if err != nil {
return err
}
library.Author, err = readString()
if err != nil {
return err
}
library.Maintainer, err = readString()
if err != nil {
return err
}
library.Sentence, err = readString()
if err != nil {
return err
}
library.Paragraph, err = readString()
if err != nil {
return err
}
library.Website, err = readString()
if err != nil {
return err
}
library.Category, err = readString()
if err != nil {
return err
}
library.Architectures, err = readStringArray()
if err != nil {
return err
}
library.Types, err = readStringArray()
if err != nil {
return err
}
installDir, err := readString()
if err != nil {
return err
}
library.InstallDir = paths.New(installDir)
library.DirName, err = readString()
if err != nil {
return err
}
sourceDir, err := readString()
library.SourceDir = paths.New(sourceDir)
if err != nil {
return err
}
utilityDir, err := readString()
if err != nil {
return err
}
library.UtilityDir = paths.New(utilityDir)
var location int32
if err := binary.Read(in, binary.NativeEndian, &location); err != nil {
return err
}
library.Location = LibraryLocation(location)
// library.ContainerPlatform *cores.PlatformRelease `json:""`
var layout int32
if err := binary.Read(in, binary.NativeEndian, &layout); err != nil {
return err
}
library.Layout = LibraryLayout(layout)
if err := binary.Read(in, binary.NativeEndian, &library.DotALinkage); err != nil {
return err
}
if err := binary.Read(in, binary.NativeEndian, &library.Precompiled); err != nil {
return err
}
if err := binary.Read(in, binary.NativeEndian, &library.PrecompiledWithSources); err != nil {
return err
}
library.LDflags, err = readString()
if err != nil {
return err
}
if err := binary.Read(in, binary.NativeEndian, &library.IsLegacy); err != nil {
return err
}
if err := binary.Read(in, binary.NativeEndian, &library.InDevelopment); err != nil {
return err
}
version, err := readString()
if err != nil {
return err
}
library.Version = semver.MustParse(version)
library.License, err = readString()
if err != nil {
return err
}
// props, err := readStringArray()
// if err != nil {
// return err
// }
// library.Properties, err = properties.LoadFromSlice(props)
// if err != nil {
// return err
// }
examples, err := readStringArray()
if err != nil {
return err
}
library.Examples = paths.NewPathList(examples...)
library.declaredHeaders, err = readStringArray()
if err != nil {
return err
}
library.sourceHeaders, err = readStringArray()
if err != nil {
return err
}
library.CompatibleWith, err = readMap()
if err != nil {
return err
}
return nil
}

// ToRPCLibrary converts this library into an rpc.Library
func (library *Library) ToRPCLibrary() (*rpc.Library, error) {
pathOrEmpty := func(p *paths.Path) string {
Expand Down
86 changes: 69 additions & 17 deletions internal/arduino/libraries/librariesmanager/librariesmanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package librariesmanager

import (
"encoding/binary"
"errors"
"fmt"
"os"
Expand Down Expand Up @@ -201,30 +202,81 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*

librariesDir.scanned = true

var libDirs paths.PathList
if librariesDir.IsSingleLibrary {
libDirs.Add(librariesDir.Path)
var loadedLibs []*libraries.Library
if cacheFilePath := librariesDir.Path.Join("libraries-loader-cache"); cacheFilePath.Exist() {
logrus.WithField("file", cacheFilePath).Info("Using library cache")

// Load lib cache
cache, err := cacheFilePath.Open()
if err != nil {
s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
defer cache.Close()
var n int32
if err := binary.Read(cache, binary.NativeEndian, &n); err != nil {
s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
loadedLibs = make([]*libraries.Library, n)
for i := range loadedLibs {
var lib libraries.Library
if err := lib.UnmarshalBinary(cache); err != nil {
s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
loadedLibs[i] = &lib
}
} else {
d, err := librariesDir.Path.ReadDir()
if os.IsNotExist(err) {
return statuses
var libDirs paths.PathList
if librariesDir.IsSingleLibrary {
libDirs.Add(librariesDir.Path)
} else {
d, err := librariesDir.Path.ReadDir()
if os.IsNotExist(err) {
return statuses
}
if err != nil {
s := status.Newf(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err))
return append(statuses, s)
}
d.FilterDirs()
d.FilterOutHiddenFiles()
libDirs = d
}

for _, libDir := range libDirs {
library, err := libraries.Load(libDir, librariesDir.Location)
if err != nil {
s := status.Newf(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err))
statuses = append(statuses, s)
continue
}
loadedLibs = append(loadedLibs, library)
}

// Write lib cache
cache, err := cacheFilePath.Create()
if err != nil {
s := status.Newf(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err))
s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
defer cache.Close()
if err := binary.Write(cache, binary.NativeEndian, int32(len(loadedLibs))); err != nil {
cacheFilePath.Remove()
s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
d.FilterDirs()
d.FilterOutHiddenFiles()
libDirs = d
for _, lib := range loadedLibs {
if _, err := cache.Write(lib.MarshalBinary()); err != nil {
cacheFilePath.Remove()
s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err)
return append(statuses, s)
}
}
}

for _, libDir := range libDirs {
library, err := libraries.Load(libDir, librariesDir.Location)
if err != nil {
s := status.Newf(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err))
statuses = append(statuses, s)
continue
}
for _, library := range loadedLibs {
library.ContainerPlatform = librariesDir.PlatformRelease
alternatives := lm.libraries[library.Name]
alternatives.Add(library)
Expand Down

0 comments on commit cbe2a4d

Please sign in to comment.