Skip to content

Commit

Permalink
Accept incoming changes
Browse files Browse the repository at this point in the history 
Signed-off-by: wayner0628 <[email protected]>
  • Loading branch information
@wayner0628
wayner0628 committed Sep 15, 2024
1 parent 96c4177 commit ad12330
Show file tree
Hide file tree
Showing 6 changed files with 0 additions and 193 deletions.
3 changes: 0 additions & 3 deletions flytepropeller/pkg/utils/failing_datastore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ func TestFailingRawStore(t *testing.T) {
c := f.GetBaseContainerFQN(ctx)
assert.Equal(t, storage.DataReference(""), c)

_, err = f.GetItems(ctx, "")
assert.Error(t, err)

_, err = f.ReadRaw(ctx, "")
assert.Error(t, err)

Expand Down
9 changes: 0 additions & 9 deletions flytestdlib/storage/cached_rawstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,6 @@ func (s *cachedRawStore) Head(ctx context.Context, reference DataReference) (Met
return s.RawStore.Head(ctx, reference)
}

// GetItems retrieves the paths of all items from the Blob store or an error
func (s *cachedRawStore) GetItems(ctx context.Context, reference DataReference) ([]string, error) {
_, span := otelutils.NewSpan(ctx, otelutils.BlobstoreClientTracer, "flytestdlib.storage.cachedRawStore/GetItems")
defer span.End()

// freecache does not support full cache scanning
return s.RawStore.GetItems(ctx, reference)
}

// ReadRaw retrieves a byte array from the Blob store or an error
func (s *cachedRawStore) ReadRaw(ctx context.Context, reference DataReference) (io.ReadCloser, error) {
ctx, span := otelutils.NewSpan(ctx, otelutils.BlobstoreClientTracer, "flytestdlib.storage.cachedRawStore/ReadRaw")
Expand Down
12 changes: 0 additions & 12 deletions flytestdlib/storage/cached_rawstore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,18 +173,6 @@ func TestCachedRawStore(t *testing.T) {
assert.False(t, m.Exists())
})

t.Run("Get Items", func(t *testing.T) {
items, err := cStore.GetItems(ctx, k1)
assert.NoError(t, err)
assert.Equal(t, 1, len(items))
})

t.Run("No Items", func(t *testing.T) {
items, err := cStore.GetItems(ctx, k2)
assert.Error(t, err)
assert.Equal(t, 0, len(items))
})

t.Run("ReadCachePopulate", func(t *testing.T) {
o, err := cStore.ReadRaw(ctx, k1)
assert.NoError(t, err)
Expand Down
1 change: 0 additions & 1 deletion flytestdlib/storage/mem_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"io"
"io/ioutil"
"os"
"strings"
)

type rawFile = []byte
Expand Down
38 changes: 0 additions & 38 deletions flytestdlib/storage/mem_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,44 +29,6 @@ func TestInMemoryStore_Head(t *testing.T) {
})
}

func TestInMemoryStore_GetItems(t *testing.T) {
t.Run("Nil Path", func(t *testing.T) {
s, err := NewInMemoryRawStore(context.TODO(), &Config{}, metrics)
assert.NoError(t, err)

items, err := s.GetItems(context.TODO(), DataReference("hello"))
assert.Error(t, err)
assert.Nil(t, items)
})

t.Run("No Items", func(t *testing.T) {
s, err := NewInMemoryRawStore(context.TODO(), &Config{}, metrics)
assert.NoError(t, err)
err = s.WriteRaw(context.TODO(), DataReference("folder"), 0, Options{}, bytes.NewReader([]byte{}))
assert.NoError(t, err)

items, err := s.GetItems(context.TODO(), DataReference("folder"))
assert.Error(t, err)
assert.Nil(t, items)
})

t.Run("Existing Items", func(t *testing.T) {
s, err := NewInMemoryRawStore(context.TODO(), &Config{}, metrics)
assert.NoError(t, err)
err = s.WriteRaw(context.TODO(), DataReference("folder/file1"), 0, Options{}, bytes.NewReader([]byte{}))
assert.NoError(t, err)

err = s.WriteRaw(context.TODO(), DataReference("folder/file2"), 0, Options{}, bytes.NewReader([]byte{}))
assert.NoError(t, err)

items, err := s.GetItems(context.TODO(), DataReference("folder"))
assert.NoError(t, err)
assert.Equal(t, 2, len(items))
assert.Equal(t, "folder/file1", items[0])
assert.Equal(t, "folder/file2", items[1])
})
}

func TestInMemoryStore_ReadRaw(t *testing.T) {
t.Run("Empty store", func(t *testing.T) {
s, err := NewInMemoryRawStore(context.TODO(), &Config{}, metrics)
Expand Down
130 changes: 0 additions & 130 deletions flytestdlib/storage/stow_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,136 +257,6 @@ func TestStowStore_CreateSignedURL(t *testing.T) {
})
}

func TestStowStore_GetItems(t *testing.T) {
const container = "container"

t.Run("Correct Query with Items", func(t *testing.T) {
ctx := context.Background()
fn := fQNFn["s3"]
s, err := NewStowRawStore(fn(container), &mockStowLoc{
ContainerCb: func(id string) (stow.Container, error) {
if id == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
CreateContainerCb: func(name string) (stow.Container, error) {
if name == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
}, nil, false, metrics)
assert.NoError(t, err)
writeTestFile(ctx, t, s, "s3://container/folder/file1")
writeTestFile(ctx, t, s, "s3://container/folder/file2")
dataReference := DataReference("s3://container/folder")
items, err := s.GetItems(ctx, dataReference)
assert.NoError(t, err)
assert.Equal(t, 2, len(items))
assert.Equal(t, "s3://container/folder/file1", items[0])
assert.Equal(t, "s3://container/folder/file2", items[1])
})

t.Run("Empty Reference", func(t *testing.T) {
ctx := context.Background()
fn := fQNFn["s3"]
s, err := NewStowRawStore(fn(container), &mockStowLoc{
ContainerCb: func(id string) (stow.Container, error) {
if id == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
CreateContainerCb: func(name string) (stow.Container, error) {
if name == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
}, nil, false, metrics)
assert.NoError(t, err)
dataReference := DataReference("s3://container/folder")
items, err := s.GetItems(ctx, dataReference)
assert.NoError(t, err)
assert.Equal(t, 0, len(items))
})

t.Run("Bad Reference", func(t *testing.T) {
ctx := context.Background()
fn := fQNFn["s3"]
s, err := NewStowRawStore(fn(container), &mockStowLoc{
ContainerCb: func(id string) (stow.Container, error) {
if id == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
CreateContainerCb: func(name string) (stow.Container, error) {
if name == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
}, nil, false, metrics)
assert.NoError(t, err)
writeTestFile(ctx, t, s, "s3://container/folder/file1")
writeTestFile(ctx, t, s, "s3://container/folder/file2")
dataReference := DataReference("s3://container/bad-folder")
items, err := s.GetItems(ctx, dataReference)
assert.NoError(t, err)
assert.Equal(t, 0, len(items))
})

t.Run("Broken Reference", func(t *testing.T) {
ctx := context.Background()
fn := fQNFn["s3"]
s, err := NewStowRawStore(fn(container), &mockStowLoc{
ContainerCb: func(id string) (stow.Container, error) {
if id == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
CreateContainerCb: func(name string) (stow.Container, error) {
if name == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
}, nil, false, metrics)
assert.NoError(t, err)
dataReference := DataReference("s3://")
items, err := s.GetItems(ctx, dataReference)
assert.Error(t, err)
assert.Nil(t, items)
})

t.Run("Non-exist Container", func(t *testing.T) {
ctx := context.Background()
fn := fQNFn["s3"]
s, err := NewStowRawStore(fn(container), &mockStowLoc{
ContainerCb: func(id string) (stow.Container, error) {
if id == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
CreateContainerCb: func(name string) (stow.Container, error) {
if name == container {
return newMockStowContainer(container), nil
}
return nil, fmt.Errorf("container is not supported")
},
}, nil, false, metrics)
assert.NoError(t, err)
dataReference := DataReference("s3://bad-container/folder")
items, err := s.GetItems(ctx, dataReference)
assert.Error(t, err)
assert.Nil(t, items)
})
}

func TestStowStore_ReadRaw(t *testing.T) {
const container = "container"
t.Run("Happy Path", func(t *testing.T) {
Expand Down

0 comments on commit ad12330

Please sign in to comment.