Skip to content

Commit

Permalink
Move many methods out of NativeLeaf
Browse files Browse the repository at this point in the history 
These are all operations that are used outside of the VFS to extract
data out of leaf nodes. By implementing all of these operations through
VirtualApply(), we can keep the VFS API decoupled from the rest of
Buildbarn.
  • Loading branch information
@EdSchouten
EdSchouten committed Sep 18, 2024
1 parent 4a8c992 commit 1d66983
Show file tree
Hide file tree
Showing 11 changed files with 320 additions and 246 deletions.
17 changes: 15 additions & 2 deletions pkg/builder/virtual_build_directory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,16 @@ func (d *virtualBuildDirectory) UploadFile(ctx context.Context, name path.Compon
return digest.BadDigest, err
}
if _, leaf := child.GetPair(); leaf != nil {
return leaf.UploadFile(ctx, d.options.contentAddressableStorage, digestFunction, writableFileUploadDelay)
p := virtual.VirtualApplyUploadFile{
Context: ctx,
ContentAddressableStorage: d.options.contentAddressableStorage,
DigestFunction: digestFunction,
WritableFileUploadDelay: writableFileUploadDelay,
}
if !child.GetNode().VirtualApply(&p) {
panic("build directory contains leaves that don't handle VirtualApplyUploadFile")
}
return p.Digest, p.Err
}
return digest.BadDigest, syscall.EISDIR
}
Expand Down Expand Up @@ -156,7 +165,11 @@ func (d *virtualBuildDirectory) Readlink(name path.Component) (path.Parser, erro
return nil, err
}
if _, leaf := child.GetPair(); leaf != nil {
return leaf.Readlink()
p := virtual.VirtualApplyReadlink{}
if !child.GetNode().VirtualApply(&p) {
panic("build directory contains leaves that don't handle VirtualApplyReadlink")
}
return p.Target, p.Err
}
return nil, syscall.EISDIR
}
58 changes: 31 additions & 27 deletions pkg/filesystem/virtual/base_symlink_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (

remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
"github.com/buildbarn/bb-remote-execution/pkg/proto/bazeloutputservice"
"github.com/buildbarn/bb-remote-execution/pkg/proto/outputpathpersistency"
"github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/filesystem"
"github.com/buildbarn/bb-storage/pkg/filesystem/path"

Expand All @@ -30,15 +28,15 @@ type symlink struct {
target []byte
}

func (f symlink) Readlink() (path.Parser, error) {
func (f symlink) readlinkParser() (path.Parser, error) {
if !utf8.Valid(f.target) {
return nil, status.Error(codes.InvalidArgument, "Symbolic link contents are not valid UTF-8")
}
return path.UNIXFormat.NewParser(string(f.target)), nil
}

func (f symlink) readlinkString() (string, error) {
targetParser, err := f.Readlink()
targetParser, err := f.readlinkParser()
if err != nil {
return "", err
}
Expand All @@ -49,29 +47,6 @@ func (f symlink) readlinkString() (string, error) {
return targetPath.GetUNIXString(), nil
}

func (f symlink) GetBazelOutputServiceStat(digestFunction *digest.Function) (*bazeloutputservice.BatchStatResponse_Stat, error) {
target, err := f.readlinkString()
if err != nil {
return nil, err
}
return &bazeloutputservice.BatchStatResponse_Stat{
Type: &bazeloutputservice.BatchStatResponse_Stat_Symlink_{
Symlink: &bazeloutputservice.BatchStatResponse_Stat_Symlink{
Target: target,
},
},
}, nil
}

func (f symlink) AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component) {
if target, err := f.readlinkString(); err == nil {
directory.Symlinks = append(directory.Symlinks, &remoteexecution.SymlinkNode{
Name: name.String(),
Target: target,
})
}
}

func (f symlink) VirtualGetAttributes(ctx context.Context, requested AttributesMask, attributes *Attributes) {
attributes.SetChangeID(0)
attributes.SetFileType(filesystem.FileTypeSymlink)
Expand All @@ -90,3 +65,32 @@ func (f symlink) VirtualSetAttributes(ctx context.Context, in *Attributes, reque
f.VirtualGetAttributes(ctx, requested, out)
return StatusOK
}

func (f symlink) VirtualApply(data any) bool {
switch p := data.(type) {
case *VirtualApplyReadlink:
p.Target, p.Err = f.readlinkParser()
case *VirtualApplyGetBazelOutputServiceStat:
if target, err := f.readlinkString(); err == nil {
p.Stat = &bazeloutputservice.BatchStatResponse_Stat{
Type: &bazeloutputservice.BatchStatResponse_Stat_Symlink_{
Symlink: &bazeloutputservice.BatchStatResponse_Stat_Symlink{
Target: target,
},
},
}
} else {
p.Err = err
}
case *VirtualApplyAppendOutputPathPersistencyDirectoryNode:
if target, err := f.readlinkString(); err == nil {
p.Directory.Symlinks = append(p.Directory.Symlinks, &remoteexecution.SymlinkNode{
Name: p.Name.String(),
Target: target,
})
}
default:
return f.placeholderFile.VirtualApply(data)
}
return true
}
114 changes: 60 additions & 54 deletions pkg/filesystem/virtual/blob_access_cas_file_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,9 @@ import (
remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
"github.com/buildbarn/bb-remote-execution/pkg/proto/bazeloutputservice"
bazeloutputservicerev2 "github.com/buildbarn/bb-remote-execution/pkg/proto/bazeloutputservice/rev2"
"github.com/buildbarn/bb-remote-execution/pkg/proto/outputpathpersistency"
"github.com/buildbarn/bb-storage/pkg/blobstore"
"github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/filesystem"
"github.com/buildbarn/bb-storage/pkg/filesystem/path"
"github.com/buildbarn/bb-storage/pkg/util"

"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -66,53 +64,49 @@ func (f *blobAccessCASFile) Link() Status {
return StatusOK
}

func (f *blobAccessCASFile) Readlink() (path.Parser, error) {
return nil, syscall.EINVAL
}

func (f *blobAccessCASFile) Unlink() {
}

func (f *blobAccessCASFile) UploadFile(ctx context.Context, contentAddressableStorage blobstore.BlobAccess, digestFunction digest.Function, writableFileUploadDelay <-chan struct{}) (digest.Digest, error) {
// This file is already backed by the Content Addressable
// Storage. There is thus no need to upload it once again.
//
// The client that created this build action already called
// FindMissingBlobs() on this file, so there's also a high
// degree of certainty that this file won't disappear from the
// Content Addressable Storage any time soon.
return f.digest, nil
}

func (f *blobAccessCASFile) GetContainingDigests() digest.Set {
return f.digest.ToSingletonSet()
}

func (f *blobAccessCASFile) GetBazelOutputServiceStat(digestFunction *digest.Function) (*bazeloutputservice.BatchStatResponse_Stat, error) {
// Assume that the file uses the same hash algorithm as
// the provided digest function. Incompatible files are
// removed from storage at the start of the build.
locator, err := anypb.New(&bazeloutputservicerev2.FileArtifactLocator{
Digest: f.digest.GetProto(),
})
if err != nil {
return nil, status.Error(codes.Internal, "Failed to marshal locator")
}
return &bazeloutputservice.BatchStatResponse_Stat{
Type: &bazeloutputservice.BatchStatResponse_Stat_File_{
File: &bazeloutputservice.BatchStatResponse_Stat_File{
Locator: locator,
},
},
}, nil
}

func (f *blobAccessCASFile) VirtualAllocate(off, size uint64) Status {
return StatusErrWrongType
}

func (blobAccessCASFile) VirtualApply(data any) bool {
return false
func (f *blobAccessCASFile) virtualApplyCommon(data any) bool {
switch p := data.(type) {
case *VirtualApplyReadlink:
p.Err = syscall.EINVAL
case *VirtualApplyUploadFile:
// This file is already backed by the Content Addressable
// Storage. There is thus no need to upload it once again.
//
// The client that created this build action already called
// FindMissingBlobs() on this file, so there's also a high
// degree of certainty that this file won't disappear from the
// Content Addressable Storage any time soon.
p.Digest = f.digest
case *VirtualApplyGetContainingDigests:
p.ContainingDigests = f.digest.ToSingletonSet()
case *VirtualApplyGetBazelOutputServiceStat:
// Assume that the file uses the same hash algorithm as
// the provided digest function. Incompatible files are
// removed from storage at the start of the build.
if locator, err := anypb.New(&bazeloutputservicerev2.FileArtifactLocator{
Digest: f.digest.GetProto(),
}); err == nil {
p.Stat = &bazeloutputservice.BatchStatResponse_Stat{
Type: &bazeloutputservice.BatchStatResponse_Stat_File_{
File: &bazeloutputservice.BatchStatResponse_Stat_File{
Locator: locator,
},
},
}
} else {
p.Err = status.Error(codes.Internal, "Failed to marshal locator")
}
default:
return false
}
return true
}

func (f *blobAccessCASFile) virtualGetAttributesCommon(attributes *Attributes) {
Expand Down Expand Up @@ -183,12 +177,18 @@ type regularBlobAccessCASFile struct {
blobAccessCASFile
}

func (f *regularBlobAccessCASFile) AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component) {
directory.Files = append(directory.Files, &remoteexecution.FileNode{
Name: name.String(),
Digest: f.digest.GetProto(),
IsExecutable: false,
})
func (f *regularBlobAccessCASFile) VirtualApply(data any) bool {
switch p := data.(type) {
case *VirtualApplyAppendOutputPathPersistencyDirectoryNode:
p.Directory.Files = append(p.Directory.Files, &remoteexecution.FileNode{
Name: p.Name.String(),
Digest: f.digest.GetProto(),
IsExecutable: false,
})
default:
return f.virtualApplyCommon(data)
}
return true
}

func (f *regularBlobAccessCASFile) VirtualGetAttributes(ctx context.Context, requested AttributesMask, attributes *Attributes) {
Expand Down Expand Up @@ -218,12 +218,18 @@ type executableBlobAccessCASFile struct {
blobAccessCASFile
}

func (f *executableBlobAccessCASFile) AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component) {
directory.Files = append(directory.Files, &remoteexecution.FileNode{
Name: name.String(),
Digest: f.digest.GetProto(),
IsExecutable: true,
})
func (f *executableBlobAccessCASFile) VirtualApply(data any) bool {
switch p := data.(type) {
case *VirtualApplyAppendOutputPathPersistencyDirectoryNode:
p.Directory.Files = append(p.Directory.Files, &remoteexecution.FileNode{
Name: p.Name.String(),
Digest: f.digest.GetProto(),
IsExecutable: true,
})
default:
return f.virtualApplyCommon(data)
}
return true
}

func (f *executableBlobAccessCASFile) VirtualGetAttributes(ctx context.Context, requested AttributesMask, attributes *Attributes) {
Expand Down
23 changes: 17 additions & 6 deletions pkg/filesystem/virtual/blob_access_cas_file_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ func TestBlobAccessCASFileFactoryGetContainingDigests(t *testing.T) {
SetSizeBytes(400),
&out)

require.Equal(t, digest.ToSingletonSet(), f.GetContainingDigests())
p := virtual.VirtualApplyGetContainingDigests{}
require.True(t, f.VirtualApply(&p))
require.Equal(t, digest.ToSingletonSet(), p.ContainingDigests)
}

func TestBlobAccessCASFileFactoryGetBazelOutputServiceStat(t *testing.T) {
Expand Down Expand Up @@ -129,8 +131,11 @@ func TestBlobAccessCASFileFactoryGetBazelOutputServiceStat(t *testing.T) {
// need to perform any I/O, as the digest is already embedded in
// the file.
digestFunction := digest.GetDigestFunction()
fileStatus, err := f.GetBazelOutputServiceStat(&digestFunction)
require.NoError(t, err)
p := virtual.VirtualApplyGetBazelOutputServiceStat{
DigestFunction: &digestFunction,
}
require.True(t, f.VirtualApply(&p))
require.NoError(t, p.Err)
locator, err := anypb.New(&bazeloutputservicerev2.FileArtifactLocator{
Digest: &remoteexecution.Digest{
Hash: "8b1a9953c4611296a827abf8c47804d7",
Expand All @@ -144,7 +149,7 @@ func TestBlobAccessCASFileFactoryGetBazelOutputServiceStat(t *testing.T) {
Locator: locator,
},
},
}, fileStatus)
}, p.Stat)
}

func TestBlobAccessCASFileFactoryAppendOutputPathPersistencyDirectoryNode(t *testing.T) {
Expand Down Expand Up @@ -184,8 +189,14 @@ func TestBlobAccessCASFileFactoryAppendOutputPathPersistencyDirectoryNode(t *tes
&out2)

var directory outputpathpersistency.Directory
f1.AppendOutputPathPersistencyDirectoryNode(&directory, path.MustNewComponent("hello"))
f2.AppendOutputPathPersistencyDirectoryNode(&directory, path.MustNewComponent("world"))
require.True(t, f1.VirtualApply(&virtual.VirtualApplyAppendOutputPathPersistencyDirectoryNode{
Directory: &directory,
Name: path.MustNewComponent("hello"),
}))
require.True(t, f2.VirtualApply(&virtual.VirtualApplyAppendOutputPathPersistencyDirectoryNode{
Directory: &directory,
Name: path.MustNewComponent("world"),
}))
testutil.RequireEqualProto(t, &outputpathpersistency.Directory{
Files: []*remoteexecution.FileNode{
{
Expand Down
38 changes: 0 additions & 38 deletions pkg/filesystem/virtual/native_leaf.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,5 @@
package virtual

import (
"context"

"github.com/buildbarn/bb-remote-execution/pkg/proto/bazeloutputservice"
"github.com/buildbarn/bb-remote-execution/pkg/proto/outputpathpersistency"
"github.com/buildbarn/bb-storage/pkg/blobstore"
"github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/filesystem/path"
)

// NativeLeaf objects are non-directory nodes that can be placed in a
// PrepopulatedDirectory.
type NativeLeaf interface {
Expand All @@ -21,32 +11,4 @@ type NativeLeaf interface {
// nodes that have been removed concurrently.
Link() Status
Unlink()

// Additional operations that are used by consumers of
// PrepopulatedDirectory.
//
// TODO: Remove these once Go supports generics. We could turn
// PrepopulatedDirectory and InitialContentsFetcher into
// parameterized types, where the leaves could be any type
// that's based on NativeLeaf.
Readlink() (path.Parser, error)
UploadFile(ctx context.Context, contentAddressableStorage blobstore.BlobAccess, digestFunction digest.Function, writableFileUploadDelay <-chan struct{}) (digest.Digest, error)
// GetContainingDigests() returns a set of digests of objects in
// the Content Addressable Storage that back the contents of
// this file.
//
// The set returned by this function may be passed to
// ContentAddressableStorage.FindMissingBlobs() to check whether
// the file still exists in its entirety, and to prevent that
// the file is removed in the nearby future.
GetContainingDigests() digest.Set
// GetBazelOutputServiceStat() returns the status of the leaf
// node in the form of a Status message that is used by the
// Bazel Output Service protocol.
GetBazelOutputServiceStat(digestFunction *digest.Function) (*bazeloutputservice.BatchStatResponse_Stat, error)
// AppendOutputPathPersistencyDirectoryNode() appends a FileNode
// or SymlinkNode entry to a Directory message that is used to
// persist the state of a Bazel Output Service output path to
// disk.
AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component)
}
Loading

0 comments on commit 1d66983

Please sign in to comment.