diff --git a/.github/services/swift/ceph_rados_swift/action.yml b/.github/services/swift/ceph_rados_swift/action.yml index 649ce2bdda1d..a4665212229d 100644 --- a/.github/services/swift/ceph_rados_swift/action.yml +++ b/.github/services/swift/ceph_rados_swift/action.yml @@ -44,7 +44,6 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV - OPENDAL_DISABLE_RANDOM_ROOT=true OPENDAL_SWIFT_CONTAINER=testing OPENDAL_SWIFT_ROOT=/ EOF diff --git a/.github/services/swift/swift/action.yml b/.github/services/swift/swift/action.yml index e7cfb4b2856a..7dc7e4ff1e1f 100644 --- a/.github/services/swift/swift/action.yml +++ b/.github/services/swift/swift/action.yml @@ -41,7 +41,6 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV - OPENDAL_DISABLE_RANDOM_ROOT=true OPENDAL_SWIFT_CONTAINER=testing OPENDAL_SWIFT_ROOT=/ EOF diff --git a/bindings/cpp/tests/basic_test.cpp b/bindings/cpp/tests/basic_test.cpp index c6fc37a0d160..8e82eb5a931c 100644 --- a/bindings/cpp/tests/basic_test.cpp +++ b/bindings/cpp/tests/basic_test.cpp @@ -77,8 +77,13 @@ TEST_F(OpendalTest, BasicTest) { auto list_file_path = dir_path + file_path; op.write(list_file_path, data); auto entries = op.list(dir_path); - EXPECT_EQ(entries.size(), 1); - EXPECT_EQ(entries[0].path, list_file_path); + EXPECT_EQ(entries.size(), 2); + std::set paths; + for (const auto &entry : entries) { + paths.insert(entry.path); + } + EXPECT_TRUE(paths.find(dir_path) != paths.end()); + EXPECT_TRUE(paths.find(list_file_path) != paths.end()); // remove op.remove(file_path_renamed); @@ -133,23 +138,23 @@ TEST_F(OpendalTest, ReaderTest) { } TEST_F(OpendalTest, ListerTest) { - op.create_dir("test_dir/"); - op.write("test_dir/test1", {1, 2, 3}); - op.write("test_dir/test2", {4, 5, 6}); + std::string dir_path = "test_dir/"; + op.create_dir(dir_path); + auto test1_path = dir_path + "test1"; + op.write(test1_path, {1, 2, 3}); + auto test2_path = dir_path + "test2"; + op.write(test2_path, {4, 5, 6}); - int size = 0; auto lister = op.lister("test_dir/"); + + std::set paths; for (const auto &entry : lister) { - EXPECT_TRUE(entry.path.find("test_dir/test") == 0); - size += 1; + paths.insert(entry.path); } - EXPECT_EQ(size, 2); - - lister = op.lister("test_dir/"); - std::vector paths(lister.begin(), lister.end()); - EXPECT_EQ(paths.size(), 2); - EXPECT_EQ(paths[0].path, "test_dir/test1"); - EXPECT_EQ(paths[1].path, "test_dir/test2"); + EXPECT_EQ(paths.size(), 3); + EXPECT_TRUE(paths.find(dir_path) != paths.end()); + EXPECT_TRUE(paths.find(test1_path) != paths.end()); + EXPECT_TRUE(paths.find(test2_path) != paths.end()); } int main(int argc, char **argv) { diff --git a/bindings/go/tests/behavior_tests/list_test.go b/bindings/go/tests/behavior_tests/list_test.go index e31f8b212e04..f4f26acf4c35 100644 --- a/bindings/go/tests/behavior_tests/list_test.go +++ b/bindings/go/tests/behavior_tests/list_test.go @@ -115,6 +115,7 @@ func testListRichDir(assert *require.Assertions, op *opendal.Operator, fixture * } assert.Nil(obs.Error()) + expected = append(expected, parent) slices.Sort(expected) slices.Sort(actual) @@ -133,10 +134,12 @@ func testListEmptyDir(assert *require.Assertions, op *opendal.Operator, fixture for obs.Next() { entry := obs.Entry() paths = append(paths, entry.Path()) + assert.Equal(dir, entry.Path()) } assert.Nil(obs.Error()) - assert.Equal(0, len(paths), "dir should only return empty") + assert.Equal(1, len(paths), "dir should only return itself") + paths = nil obs, err = op.List(strings.TrimSuffix(dir, "/")) assert.Nil(err) defer obs.Close() @@ -211,6 +214,7 @@ func testListNestedDir(assert *require.Assertions, op *opendal.Operator, fixture defer obs.Close() paths = nil var foundFile bool + var foundDirPath bool var foundDir bool for obs.Next() { entry := obs.Entry() @@ -218,11 +222,15 @@ func testListNestedDir(assert *require.Assertions, op *opendal.Operator, fixture if entry.Path() == filePath { foundFile = true } else if entry.Path() == dirPath { + foundDirPath = true + } else if entry.Path() == dir { foundDir = true } } assert.Nil(obs.Error()) - assert.Equal(2, len(paths), "parent should only got 2 entries") + assert.Equal(3, len(paths), "dir should only got 3 entries") + + assert.True(foundDir, "dir should be found in list") assert.True(foundFile, "file should be found in list") meta, err := op.Stat(filePath) @@ -230,7 +238,7 @@ func testListNestedDir(assert *require.Assertions, op *opendal.Operator, fixture assert.True(meta.IsFile()) assert.Equal(uint64(20), meta.ContentLength()) - assert.True(foundDir, "dir should be found in list") + assert.True(foundDirPath, "dir path should be found in list") meta, err = op.Stat(dirPath) assert.Nil(err) assert.True(meta.IsDir()) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java index 286d93a38771..bf0fd90d9258 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java @@ -33,6 +33,7 @@ import org.apache.opendal.Metadata; import org.apache.opendal.OpenDALException; import org.apache.opendal.test.condition.OpenDALExceptionCondition; +import org.assertj.core.util.Lists; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -77,24 +78,25 @@ public void testListDir() { */ @Test public void testListRichDir() { - final String parent = "test_list_rich_dir"; - asyncOp().createDir(parent + "/").join(); + final String parentPath = "test_list_rich_dir/"; + asyncOp().createDir(parentPath).join(); final List expected = new ArrayList<>(); for (int i = 0; i < 10; i++) { - expected.add(String.format("%s/file-%d", parent, i)); + expected.add(String.format("%sfile-%d", parentPath, i)); } for (String path : expected) { - asyncOp().write(path, parent).join(); + asyncOp().write(path, parentPath).join(); } - final List entries = asyncOp().list(parent + "/").join(); + final List entries = asyncOp().list(parentPath).join(); final List actual = entries.stream().map(Entry::getPath).sorted().collect(Collectors.toList()); + expected.add(parentPath); Collections.sort(expected); assertThat(actual).isEqualTo(expected); - asyncOp().removeAll(parent + "/").join(); + asyncOp().removeAll(parentPath).join(); } /** @@ -106,7 +108,9 @@ public void testListEmptyDir() { asyncOp().createDir(dir).join(); final List entries = asyncOp().list(dir).join(); - assertThat(entries).isEmpty(); + final List actual = entries.stream().map(Entry::getPath).collect(Collectors.toList()); + assertThat(actual).hasSize(1); + assertThat(actual.get(0)).isEqualTo(dir); asyncOp().delete(dir).join(); } @@ -151,9 +155,9 @@ public void testListSubDir() { public void testListNestedDir() { final String dir = String.format("%s/%s/", UUID.randomUUID(), UUID.randomUUID()); final String fileName = UUID.randomUUID().toString(); - final String filePath = String.format("%s/%s", dir, fileName); + final String filePath = String.format("%s%s", dir, fileName); final String dirName = String.format("%s/", UUID.randomUUID()); - final String dirPath = String.format("%s/%s", dir, dirName); + final String dirPath = String.format("%s%s", dir, dirName); final String content = "test_list_nested_dir"; asyncOp().createDir(dir).join(); @@ -161,18 +165,27 @@ public void testListNestedDir() { asyncOp().createDir(dirPath).join(); final List entries = asyncOp().list(dir).join(); - assertThat(entries).hasSize(2); + assertThat(entries).hasSize(3); + + final List expectedPaths = Lists.newArrayList(dir, dirPath, filePath); + Collections.sort(expectedPaths); + final List actualPaths = + entries.stream().map(Entry::getPath).sorted().collect(Collectors.toList()); + assertThat(actualPaths).isEqualTo(expectedPaths); for (Entry entry : entries) { // check file if (entry.getPath().equals(filePath)) { - Metadata metadata = entry.getMetadata(); + Metadata metadata = asyncOp().stat(filePath).join(); assertTrue(metadata.isFile()); assertThat(metadata.getContentLength()).isEqualTo(content.length()); // check dir } else if (entry.getPath().equals(dirPath)) { Metadata metadata = entry.getMetadata(); assertTrue(metadata.isDir()); + } else { + assertThat(entry.getPath()).isEqualTo(dir); + assertTrue(entry.metadata.isDir()); } } diff --git a/bindings/ocaml/test/test.ml b/bindings/ocaml/test/test.ml index f9770d9aa0ab..8e0f12fbd7af 100644 --- a/bindings/ocaml/test/test.ml +++ b/bindings/ocaml/test/test.ml @@ -83,10 +83,10 @@ let test_list test_ctxt = (Operator.write bo "/testdir/bar" (Bytes.of_string "foo"))); let array = Operator.list bo "testdir/" |> test_check_result in let actual = Array.map Operator.Entry.name array in - let expected = [| "foo"; "bar" |] in + let expected = [| "testdir/"; "foo"; "bar" |] in List.iter (Array.sort compare) [ expected; actual ]; assert_equal expected actual; - assert_equal 2 (Array.length array); + assert_equal 3 (Array.length array); () let suite = diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index d89aecb2cce6..17c0c6414a01 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -195,10 +195,7 @@ impl CompleteAccessor { if path.ends_with('/') && capability.list_with_recursive { let (_, mut l) = self .inner - .list( - path.trim_end_matches('/'), - OpList::default().with_recursive(true).with_limit(1), - ) + .list(path, OpList::default().with_recursive(true).with_limit(1)) .await?; return if oio::List::next(&mut l).await?.is_some() { @@ -246,10 +243,9 @@ impl CompleteAccessor { // Otherwise, we can simulate stat a dir path via `list`. if path.ends_with('/') && capability.list_with_recursive { - let (_, mut l) = self.inner.blocking_list( - path.trim_end_matches('/'), - OpList::default().with_recursive(true).with_limit(1), - )?; + let (_, mut l) = self + .inner + .blocking_list(path, OpList::default().with_recursive(true).with_limit(1))?; return if oio::BlockingList::next(&mut l)?.is_some() { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs index a4a878924c7c..3a7ea3525d94 100644 --- a/core/src/raw/adapters/kv/backend.rs +++ b/core/src/raw/adapters/kv/backend.rs @@ -209,8 +209,11 @@ impl KvLister { } else { EntryMode::FILE }; - - oio::Entry::new(&build_rel_path(&self.root, &v), Metadata::new(mode)) + let mut path = build_rel_path(&self.root, &v); + if path.is_empty() { + path = "/".to_string(); + } + oio::Entry::new(&path, Metadata::new(mode)) }) } } diff --git a/core/src/raw/adapters/typed_kv/api.rs b/core/src/raw/adapters/typed_kv/api.rs index 9a0949df569a..0fecb9d01299 100644 --- a/core/src/raw/adapters/typed_kv/api.rs +++ b/core/src/raw/adapters/typed_kv/api.rs @@ -39,7 +39,7 @@ use crate::Scheme; /// /// `typed_kv::Adapter` is the typed version of `kv::Adapter`. It's more /// efficient if the underlying kv service can store data with its type. For -/// example, we can store `Bytes` along with it's metadata so that we don't +/// example, we can store `Bytes` along with its metadata so that we don't /// need to serialize/deserialize it when we get it from the service. /// /// Ideally, we should use `typed_kv::Adapter` instead of `kv::Adapter` for diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs index e65ac6f4e6b8..1a04efd308fc 100644 --- a/core/src/raw/adapters/typed_kv/backend.rs +++ b/core/src/raw/adapters/typed_kv/backend.rs @@ -15,15 +15,14 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; -use std::vec::IntoIter; - use super::Adapter; use super::Value; use crate::raw::oio::HierarchyLister; use crate::raw::oio::QueueBuf; use crate::raw::*; use crate::*; +use std::sync::Arc; +use std::vec::IntoIter; /// The typed kv backend which implements Accessor for typed kv adapter. #[derive(Debug, Clone)] @@ -211,8 +210,11 @@ impl KvLister { } else { EntryMode::FILE }; - - oio::Entry::new(&build_rel_path(&self.root, &v), Metadata::new(mode)) + let mut path = build_rel_path(&self.root, &v); + if path.is_empty() { + path = "/".to_string(); + } + oio::Entry::new(&path, Metadata::new(mode)) }) } } diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs index 8fc0b81ef0a0..7d756030da0b 100644 --- a/core/src/raw/http_util/client.rs +++ b/core/src/raw/http_util/client.rs @@ -122,7 +122,7 @@ impl HttpClient { // Get content length from header so that we can check it. // // - If the request method is HEAD, we will ignore content length. - // - If response contains content_encoding, we should omit it's content length. + // - If response contains content_encoding, we should omit its content length. let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() { None } else { diff --git a/core/src/raw/oio/list/flat_list.rs b/core/src/raw/oio/list/flat_list.rs index a399539b9cdf..7a7096308762 100644 --- a/core/src/raw/oio/list/flat_list.rs +++ b/core/src/raw/oio/list/flat_list.rs @@ -56,7 +56,6 @@ use crate::*; /// always output directly while listing. pub struct FlatLister { acc: A, - root: String, next_dir: Option, active_lister: Vec<(Option, L)>, @@ -79,7 +78,6 @@ where pub fn new(acc: A, path: &str) -> FlatLister { FlatLister { acc, - root: path.to_string(), next_dir: Some(oio::Entry::new(path, Metadata::new(EntryMode::DIR))), active_lister: vec![], } @@ -105,25 +103,22 @@ where match lister.next().await? { Some(v) if v.mode().is_dir() => { - self.next_dir = Some(v); - continue; + // should not loop itself again + if v.path() != de.as_ref().expect("de should not be none here").path() { + self.next_dir = Some(v); + continue; + } } Some(v) => return Ok(Some(v)), - None => { - match de.take() { - Some(de) => { - // Only push entry if it's not root dir - if de.path() != self.root { - return Ok(Some(de)); - } - continue; - } - None => { - let _ = self.active_lister.pop(); - continue; - } + None => match de.take() { + Some(de) => { + return Ok(Some(de)); } - } + None => { + let _ = self.active_lister.pop(); + continue; + } + }, } } } @@ -149,25 +144,21 @@ where match lister.next()? { Some(v) if v.mode().is_dir() => { - self.next_dir = Some(v); - continue; + if v.path() != de.as_ref().expect("de should not be none here").path() { + self.next_dir = Some(v); + continue; + } } Some(v) => return Ok(Some(v)), - None => { - match de.take() { - Some(de) => { - // Only push entry if it's not root dir - if de.path() != self.root { - return Ok(Some(de)); - } - continue; - } - None => { - let _ = self.active_lister.pop(); - continue; - } + None => match de.take() { + Some(de) => { + return Ok(Some(de)); } - } + None => { + let _ = self.active_lister.pop(); + continue; + } + }, } } } diff --git a/core/src/raw/oio/list/hierarchy_list.rs b/core/src/raw/oio/list/hierarchy_list.rs index bed486ce74e6..0ebaf28aec62 100644 --- a/core/src/raw/oio/list/hierarchy_list.rs +++ b/core/src/raw/oio/list/hierarchy_list.rs @@ -69,11 +69,6 @@ impl

HierarchyLister

{ return false; } - // Dir itself should not be returned in hierarchy page. - if e.path() == self.path { - return false; - } - // Don't return already visited path. if self.visited.contains(e.path()) { return false; diff --git a/core/src/raw/oio/list/page_list.rs b/core/src/raw/oio/list/page_list.rs index 4cce1209011f..2e146e33ce64 100644 --- a/core/src/raw/oio/list/page_list.rs +++ b/core/src/raw/oio/list/page_list.rs @@ -54,11 +54,11 @@ pub struct PageContext { pub done: bool, /// token is used by underlying storage services to fetch next page. pub token: String, - /// entries is used to store entries fetched from underlying storage. + /// entries are used to store entries fetched from underlying storage. /// /// Please always reuse the same `VecDeque` to avoid unnecessary memory allocation. /// PageLister makes sure that entries is reset before calling `next_page`. Implementer - /// can calling `push_back` on `entries` directly. + /// can call `push_back` on `entries` directly. pub entries: VecDeque, } diff --git a/core/src/raw/oio/list/prefix_list.rs b/core/src/raw/oio/list/prefix_list.rs index e82eee29b17d..be6d7e9654f1 100644 --- a/core/src/raw/oio/list/prefix_list.rs +++ b/core/src/raw/oio/list/prefix_list.rs @@ -49,15 +49,6 @@ impl PrefixLister { } } -#[inline] -fn starts_with_not_eq(entry: &oio::Entry, prefix: &str) -> bool { - match entry.path().strip_prefix(prefix) { - None => false, - Some("") => false, - Some(_) => true, - } -} - impl oio::List for PrefixLister where L: oio::List, @@ -65,7 +56,7 @@ where async fn next(&mut self) -> Result> { loop { match self.lister.next().await { - Ok(Some(e)) if !starts_with_not_eq(&e, &self.prefix) => continue, + Ok(Some(e)) if !e.path().starts_with(&self.prefix) => continue, v => return v, } } @@ -79,7 +70,7 @@ where fn next(&mut self) -> Result> { loop { match self.lister.next() { - Ok(Some(e)) if !starts_with_not_eq(&e, &self.prefix) => continue, + Ok(Some(e)) if !e.path().starts_with(&self.prefix) => continue, v => return v, } } diff --git a/core/src/services/azblob/lister.rs b/core/src/services/azblob/lister.rs index 17b800be9cbb..4804b64b3305 100644 --- a/core/src/services/azblob/lister.rs +++ b/core/src/services/azblob/lister.rs @@ -17,14 +17,13 @@ use std::sync::Arc; -use bytes::Buf; -use quick_xml::de; - use super::core::AzblobCore; use super::core::ListBlobsOutput; use super::error::parse_error; use crate::raw::*; use crate::*; +use bytes::Buf; +use quick_xml::de; pub struct AzblobLister { core: Arc, @@ -83,11 +82,9 @@ impl oio::PageList for AzblobLister { } for object in output.blobs.blob { - let path = build_rel_path(&self.core.root, &object.name); - - // azblob could return the dir itself in contents. - if path == self.path || path.is_empty() { - continue; + let mut path = build_rel_path(&self.core.root, &object.name); + if path.is_empty() { + path = "/".to_string(); } let meta = Metadata::new(EntryMode::from_path(&path)) diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index 9b8bdf55e647..7178ab9974d0 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -146,7 +146,7 @@ impl typed_kv::Adapter for Adapter { if path.is_empty() { Ok(keys.collect()) } else { - Ok(keys.filter(|k| k.starts_with(path) && k != path).collect()) + Ok(keys.filter(|k| k.starts_with(path)).collect()) } } } diff --git a/core/src/services/etcd/backend.rs b/core/src/services/etcd/backend.rs index fa7fb348207c..f01103823ec7 100644 --- a/core/src/services/etcd/backend.rs +++ b/core/src/services/etcd/backend.rs @@ -388,9 +388,6 @@ impl kv::Adapter for Adapter { Error::new(ErrorKind::Unexpected, "store key is not valid utf-8 string") .set_source(err) })?; - if v == path { - continue; - } res.push(v); } diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index af1f163187c8..a8e94f7a40a4 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -369,8 +369,7 @@ impl Access for FsBackend { } }; - let rd = FsLister::new(&self.core.root, f, arg); - + let rd = FsLister::new(&self.core.root, path, f, arg); Ok((RpList::default(), Some(rd))) } @@ -539,8 +538,7 @@ impl Access for FsBackend { } }; - let rd = FsLister::new(&self.core.root, f, arg); - + let rd = FsLister::new(&self.core.root, path, f, arg); Ok((RpList::default(), Some(rd))) } diff --git a/core/src/services/fs/lister.rs b/core/src/services/fs/lister.rs index 229e7f78966d..1b8ae3bd6649 100644 --- a/core/src/services/fs/lister.rs +++ b/core/src/services/fs/lister.rs @@ -25,15 +25,18 @@ use std::path::PathBuf; pub struct FsLister

{ root: PathBuf, + current_path: Option, + rd: P, op: OpList, } impl

FsLister

{ - pub fn new(root: &Path, rd: P, arg: OpList) -> Self { + pub fn new(root: &Path, path: &str, rd: P, arg: OpList) -> Self { Self { root: root.to_owned(), + current_path: Some(path.to_string()), rd, op: arg, } @@ -47,6 +50,12 @@ unsafe impl

Sync for FsLister

{} impl oio::List for FsLister { async fn next(&mut self) -> Result> { + // since list should return path itself, we return it first + if let Some(path) = self.current_path.take() { + let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR)); + return Ok(Some(e)); + } + let Some(de) = self.rd.next_entry().await.map_err(new_std_io_error)? else { return Ok(None); }; @@ -98,6 +107,12 @@ impl oio::List for FsLister { impl oio::BlockingList for FsLister { fn next(&mut self) -> Result> { + // since list should return path itself, we return it first + if let Some(path) = self.current_path.take() { + let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR)); + return Ok(Some(e)); + } + let de = match self.rd.next() { Some(de) => de.map_err(new_std_io_error)?, None => return Ok(None), diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs index ef37a767a6ac..cd66e964f77b 100644 --- a/core/src/services/gcs/lister.rs +++ b/core/src/services/gcs/lister.rs @@ -102,9 +102,9 @@ impl oio::PageList for GcsLister { for object in output.items { // exclude the inclusive start_after itself - let path = build_rel_path(&self.core.root, &object.name); - if path == self.path || path.is_empty() { - continue; + let mut path = build_rel_path(&self.core.root, &object.name); + if path.is_empty() { + path = "/".to_string(); } if self.start_after.as_ref() == Some(&path) { continue; diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index ff240c248c28..b86be5ddfa43 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -424,7 +424,7 @@ impl Access for HdfsBackend { } }; - let rd = HdfsLister::new(&self.root, f); + let rd = HdfsLister::new(&self.root, f, path); Ok((RpList::default(), Some(rd))) } @@ -620,7 +620,7 @@ impl Access for HdfsBackend { } }; - let rd = HdfsLister::new(&self.root, f); + let rd = HdfsLister::new(&self.root, f, path); Ok((RpList::default(), Some(rd))) } diff --git a/core/src/services/hdfs/lister.rs b/core/src/services/hdfs/lister.rs index 8daebbb04f95..35be04ee497a 100644 --- a/core/src/services/hdfs/lister.rs +++ b/core/src/services/hdfs/lister.rs @@ -24,20 +24,28 @@ pub struct HdfsLister { root: String, rd: hdrs::Readdir, + + current_path: Option, } impl HdfsLister { - pub fn new(root: &str, rd: hdrs::Readdir) -> Self { + pub fn new(root: &str, rd: hdrs::Readdir, path: &str) -> Self { Self { root: root.to_string(), rd, + + current_path: Some(path.to_string()), } } } impl oio::List for HdfsLister { async fn next(&mut self) -> Result> { + if let Some(path) = self.current_path.take() { + return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR)))); + } + let de = match self.rd.next() { Some(de) => de, None => return Ok(None), @@ -63,6 +71,10 @@ impl oio::List for HdfsLister { impl oio::BlockingList for HdfsLister { fn next(&mut self) -> Result> { + if let Some(path) = self.current_path.take() { + return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR)))); + } + let de = match self.rd.next() { Some(de) => de, None => return Ok(None), diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs index ef7ca83760fc..7b95f5d68b85 100644 --- a/core/src/services/memory/backend.rs +++ b/core/src/services/memory/backend.rs @@ -146,11 +146,9 @@ impl typed_kv::Adapter for Adapter { }; inner .range(path.to_string()..right_range) - .filter(|(k, _)| k.as_str() != path) .map(|(k, _)| k.to_string()) .collect() }; - Ok(keys) } } diff --git a/core/src/services/mini_moka/backend.rs b/core/src/services/mini_moka/backend.rs index e84a82c8f38e..95d05ef4f284 100644 --- a/core/src/services/mini_moka/backend.rs +++ b/core/src/services/mini_moka/backend.rs @@ -210,7 +210,7 @@ impl typed_kv::Adapter for Adapter { if path.is_empty() { Ok(keys.collect()) } else { - Ok(keys.filter(|k| k.starts_with(path) && k != path).collect()) + Ok(keys.filter(|k| k.starts_with(path)).collect()) } } } diff --git a/core/src/services/moka/backend.rs b/core/src/services/moka/backend.rs index 1f29fd8bcf87..98d429308ea1 100644 --- a/core/src/services/moka/backend.rs +++ b/core/src/services/moka/backend.rs @@ -252,7 +252,7 @@ impl typed_kv::Adapter for Adapter { if path.is_empty() { Ok(keys.collect()) } else { - Ok(keys.filter(|k| k.starts_with(path) && k != path).collect()) + Ok(keys.filter(|k| k.starts_with(path)).collect()) } } } diff --git a/core/src/services/oss/lister.rs b/core/src/services/oss/lister.rs index 0022597a7eed..4871e9692b9e 100644 --- a/core/src/services/oss/lister.rs +++ b/core/src/services/oss/lister.rs @@ -93,9 +93,9 @@ impl oio::PageList for OssLister { } for object in output.contents { - let path = build_rel_path(&self.core.root, &object.key); - if path == self.path || path.is_empty() { - continue; + let mut path = build_rel_path(&self.core.root, &object.key); + if path.is_empty() { + path = "/".to_string(); } if self.start_after.as_ref() == Some(&path) { continue; diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs index e117b41aaee0..237db971edeb 100644 --- a/core/src/services/rocksdb/backend.rs +++ b/core/src/services/rocksdb/backend.rs @@ -199,10 +199,6 @@ impl kv::Adapter for Adapter { if !key.starts_with(path) { continue; } - // List should skip the path itself. - if key == path { - continue; - } res.push(key.to_string()); } diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs index 7d1e057fef2e..cb6640f1c5e1 100644 --- a/core/src/services/s3/lister.rs +++ b/core/src/services/s3/lister.rs @@ -68,7 +68,7 @@ impl oio::PageList for S3Lister { &ctx.token, self.delimiter, self.limit, - // State after should only be set for the first page. + // start after should only be set for the first page. if ctx.token.is_empty() { self.start_after.clone() } else { @@ -80,7 +80,6 @@ impl oio::PageList for S3Lister { if resp.status() != http::StatusCode::OK { return Err(parse_error(resp)); } - let bs = resp.into_body(); let output: ListObjectsOutput = @@ -90,7 +89,7 @@ impl oio::PageList for S3Lister { // // - Check `is_truncated` // - Check `next_continuation_token` - // - Check the length of `common_prefixes` and `contents` (very rarely case) + // - Check the length of `common_prefixes` and `contents` (very rare case) ctx.done = if let Some(is_truncated) = output.is_truncated { !is_truncated } else if let Some(next_continuation_token) = output.next_continuation_token.as_ref() { @@ -110,11 +109,9 @@ impl oio::PageList for S3Lister { } for object in output.contents { - let path = build_rel_path(&self.core.root, &object.key); - - // s3 could return the dir itself in contents. - if path == self.path || path.is_empty() { - continue; + let mut path = build_rel_path(&self.core.root, &object.key); + if path.is_empty() { + path = "/".to_string(); } let mut meta = Metadata::new(EntryMode::from_path(&path)); @@ -125,7 +122,7 @@ impl oio::PageList for S3Lister { } meta.set_content_length(object.size); - // object.last_modified provides more precious time that contains + // object.last_modified provides more precise time that contains // nanosecond, let's trim them. meta.set_last_modified(parse_datetime_from_rfc3339(object.last_modified.as_str())?); diff --git a/core/src/services/seafile/lister.rs b/core/src/services/seafile/lister.rs index 0e56de13f8c5..e5cc3fe35767 100644 --- a/core/src/services/seafile/lister.rs +++ b/core/src/services/seafile/lister.rs @@ -74,6 +74,12 @@ impl oio::PageList for SeafileLister { let infos: Vec = serde_json::from_reader(resp_body.reader()) .map_err(new_json_deserialize_error)?; + // add path itself + ctx.entries.push_back(Entry::new( + self.path.as_str(), + Metadata::new(EntryMode::DIR), + )); + for info in infos { if !info.name.is_empty() { let rel_path = diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 637164e68964..f945c56e5dd7 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -81,7 +81,7 @@ impl Configurator for SftpConfig { /// SFTP services support. (only works on unix) /// -/// If you are interested in working on windows, pl ease refer to [this](https://github.com/apache/opendal/issues/2963) issue. +/// If you are interested in working on windows, please refer to [this](https://github.com/apache/opendal/issues/2963) issue. /// Welcome to leave your comments or make contributions. /// /// Warning: Maximum number of file holdings is depending on the remote system configuration. diff --git a/core/src/services/sftp/lister.rs b/core/src/services/sftp/lister.rs index 077e123ee396..dacda8580b3a 100644 --- a/core/src/services/sftp/lister.rs +++ b/core/src/services/sftp/lister.rs @@ -54,8 +54,14 @@ impl oio::List for SftpLister { match item { Some(e) => { - if e.filename().to_str() == Some(".") || e.filename().to_str() == Some("..") { + if e.filename().to_str() == Some("..") { continue; + } else if e.filename().to_str() == Some(".") { + let mut path = self.prefix.as_str(); + if self.prefix.is_empty() { + path = "/"; + } + return Ok(Some(Entry::new(path, e.metadata().into()))); } else { return Ok(Some(map_entry(self.prefix.as_str(), e))); } @@ -66,7 +72,7 @@ impl oio::List for SftpLister { } } -fn map_entry(prefix: &str, value: DirEntry) -> oio::Entry { +fn map_entry(prefix: &str, value: DirEntry) -> Entry { let path = format!( "{}{}{}", prefix, @@ -78,5 +84,5 @@ fn map_entry(prefix: &str, value: DirEntry) -> oio::Entry { } ); - oio::Entry::new(path.as_str(), value.metadata().into()) + Entry::new(path.as_str(), value.metadata().into()) } diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs index b100c4e6877b..d64d09b77afb 100644 --- a/core/src/services/sled/backend.rs +++ b/core/src/services/sled/backend.rs @@ -139,7 +139,7 @@ impl Builder for SledBuilder { datadir: datadir_path, tree, }) - .with_root(self.config.root.as_deref().unwrap_or_default())) + .with_root(self.config.root.as_deref().unwrap_or("/"))) } } @@ -242,9 +242,6 @@ impl kv::Adapter for Adapter { Error::new(ErrorKind::Unexpected, "store key is not valid utf-8 string") .set_source(err) })?; - if v == path { - continue; - } res.push(v); } diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index 4e9538509cb4..ef70923116b8 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -347,12 +347,12 @@ impl kv::Adapter for Adapter { fn blocking_scan(&self, path: &str) -> Result> { let conn = self.pool.get().map_err(parse_r2d2_error)?; let query = format!( - "SELECT {} FROM {} WHERE `{}` LIKE $1 and `{}` <> $2", - self.key_field, self.table, self.key_field, self.key_field + "SELECT {} FROM {} WHERE `{}` LIKE $1", + self.key_field, self.table, self.key_field ); let mut statement = conn.prepare(&query).map_err(parse_rusqlite_error)?; let like_param = format!("{}%", path); - let result = statement.query(params![like_param, path]); + let result = statement.query(params![like_param]); match result { Ok(mut rows) => { diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs index 7e59e90465a1..f4662d41336b 100644 --- a/core/src/services/swift/core.rs +++ b/core/src/services/swift/core.rs @@ -107,7 +107,6 @@ impl SwiftCore { body: Buffer, ) -> Result> { let p = build_abs_path(&self.root, path); - let url = format!( "{}/{}/{}", &self.endpoint, diff --git a/core/src/services/swift/lister.rs b/core/src/services/swift/lister.rs index 50a9edff99e3..2fe05ee9541b 100644 --- a/core/src/services/swift/lister.rs +++ b/core/src/services/swift/lister.rs @@ -75,8 +75,12 @@ impl oio::PageList for SwiftLister { for status in decoded_response { let entry: oio::Entry = match status { ListOpResponse::Subdir { subdir } => { + let mut path = build_rel_path(self.core.root.as_str(), subdir.as_str()); + if path.is_empty() { + path = "/".to_string(); + } let meta = Metadata::new(EntryMode::DIR); - oio::Entry::new(&subdir, meta) + oio::Entry::with(path, meta) } ListOpResponse::FileInfo { bytes, @@ -85,12 +89,11 @@ impl oio::PageList for SwiftLister { content_type, mut last_modified, } => { - // this is the pseudo directory itself; we'll skip it. - if name == self.path { - continue; + let mut path = build_rel_path(self.core.root.as_str(), name.as_str()); + if path.is_empty() { + path = "/".to_string(); } - - let mut meta = Metadata::new(EntryMode::from_path(&name)); + let mut meta = Metadata::new(EntryMode::from_path(path.as_str())); meta.set_content_length(bytes); meta.set_content_md5(hash.as_str()); @@ -106,7 +109,7 @@ impl oio::PageList for SwiftLister { meta.set_content_type(content_type.as_str()); } - oio::Entry::with(name, meta) + oio::Entry::with(path, meta) } }; ctx.entries.push_back(entry); diff --git a/core/src/services/webdav/core.rs b/core/src/services/webdav/core.rs index fb44b950ba58..03ebac51f852 100644 --- a/core/src/services/webdav/core.rs +++ b/core/src/services/webdav/core.rs @@ -292,7 +292,7 @@ impl WebdavCore { loop { match self.webdav_stat_rooted_abs_path(path).await { - // Dir is exist, break the loop. + // Dir exists, break the loop. Ok(_) => { break; } diff --git a/core/src/services/webdav/lister.rs b/core/src/services/webdav/lister.rs index d94914dfe181..8e9601559450 100644 --- a/core/src/services/webdav/lister.rs +++ b/core/src/services/webdav/lister.rs @@ -88,11 +88,6 @@ impl oio::PageList for WebdavLister { let normalized_path = build_rel_path(&self.core.root, &path); let decoded_path = percent_decode_path(&normalized_path); - if normalized_path == self.path || decoded_path == self.path { - // WebDAV server may return the current path as an entry. - continue; - } - // HACKS! HACKS! HACKS! // // jfrog artifactory will generate a virtual checksum file for each file. diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 0354f3659d4d..38f2cd44eb5c 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -176,7 +176,7 @@ impl Builder for WebhdfsBuilder { /// /// when building backend, the built backend will check if the root directory /// exits. - /// if the directory does not exits, the directory will be automatically created + /// if the directory does not exit, the directory will be automatically created fn build(self) -> Result { debug!("start building backend: {:?}", self); diff --git a/core/src/services/webhdfs/docs.md b/core/src/services/webhdfs/docs.md index 6dce0b6e6604..779d9a468556 100644 --- a/core/src/services/webhdfs/docs.md +++ b/core/src/services/webhdfs/docs.md @@ -70,10 +70,10 @@ async fn main() -> Result<()> { // set the root for WebHDFS, all operations will happen under this root // // Note: - // if the root is not exists, the builder will automatically create the + // if the root exists, the builder will automatically create the // root directory for you // if the root exists and is a directory, the builder will continue working - // if the root exists and is a folder, the builder will fail on building backend + // if the root exists and is a file, the builder will fail on building backend .root("/path/to/dir") // set the endpoint of webhdfs namenode, controlled by dfs.namenode.http-address // default is http://127.0.0.1:9870 diff --git a/core/src/services/webhdfs/lister.rs b/core/src/services/webhdfs/lister.rs index e7f714e29a20..d714bd446f77 100644 --- a/core/src/services/webhdfs/lister.rs +++ b/core/src/services/webhdfs/lister.rs @@ -45,6 +45,10 @@ impl oio::PageList for WebhdfsLister { match resp.status() { StatusCode::OK => { ctx.done = true; + ctx.entries.push_back(oio::Entry::new( + format!("{}/", self.path).as_str(), + Metadata::new(EntryMode::DIR), + )); let bs = resp.into_body(); serde_json::from_reader::<_, FileStatusesWrapper>(bs.reader()) @@ -72,6 +76,11 @@ impl oio::PageList for WebhdfsLister { let file_statuses = directory_listing.partial_listing.file_statuses.file_status; if directory_listing.remaining_entries == 0 { + ctx.entries.push_back(oio::Entry::new( + format!("{}/", self.path).as_str(), + Metadata::new(EntryMode::DIR), + )); + ctx.done = true; } else if !file_statuses.is_empty() { ctx.token diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs index 5879d22daf40..3e5c5bab8f82 100644 --- a/core/src/types/operator/blocking_operator.rs +++ b/core/src/types/operator/blocking_operator.rs @@ -863,9 +863,6 @@ impl BlockingOperator { } } - // Remove the directory itself. - self.delete(path)?; - Ok(()) } diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index 7b4ef460d2db..cf8d65844c23 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -1452,9 +1452,6 @@ impl Operator { .await?; } - // Remove the directory itself. - self.delete(path).await?; - Ok(()) } diff --git a/core/tests/behavior/async_list.rs b/core/tests/behavior/async_list.rs index ae5c6b04ba74..221f486fe9e2 100644 --- a/core/tests/behavior/async_list.rs +++ b/core/tests/behavior/async_list.rs @@ -18,14 +18,13 @@ use std::collections::HashMap; use std::collections::HashSet; +use crate::*; use anyhow::Result; use futures::stream::FuturesUnordered; use futures::StreamExt; use futures::TryStreamExt; use log::debug; -use crate::*; - pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); @@ -188,7 +187,7 @@ pub async fn test_list_prefix(op: Operator) -> Result<()> { op.write(&path, content).await.expect("write must succeed"); - let obs = op.list(&path[..path.len() - 1]).await?; + let obs = op.list(&path).await?; assert_eq!(obs.len(), 1); assert_eq!(obs[0].path(), path); assert_eq!(obs[0].metadata().mode(), EntryMode::FILE); @@ -205,17 +204,16 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> { return Ok(()); } - op.create_dir("test_list_rich_dir/").await?; - - let mut expected: Vec = (0..=10) - .map(|num| format!("test_list_rich_dir/file-{num}")) - .collect(); + let parent = "test_list_rich_dir/"; + op.create_dir(parent).await?; + let mut expected: Vec = (0..=10).map(|num| format!("{parent}file-{num}")).collect(); for path in expected.iter() { op.write(path, "test_list_rich_dir").await?; } + expected.push(parent.to_string()); - let mut objects = op.lister_with("test_list_rich_dir/").limit(5).await?; + let mut objects = op.lister_with(parent).limit(5).await?; let mut actual = vec![]; while let Some(o) = objects.try_next().await? { let path = o.path().to_string(); @@ -228,7 +226,7 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> { // List concurrently. let mut objects = op - .lister_with("test_list_rich_dir/") + .lister_with(parent) .limit(5) .concurrent(5) .metakey(Metakey::Complete) @@ -243,23 +241,32 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> { assert_eq!(actual, expected); - op.remove_all("test_list_rich_dir/").await?; + op.remove_all(parent).await?; Ok(()) } -/// List empty dir should return nothing. +/// List empty dir should return itself. pub async fn test_list_empty_dir(op: Operator) -> Result<()> { let dir = format!("{}/", uuid::Uuid::new_v4()); op.create_dir(&dir).await.expect("write must succeed"); - // List "dir/" should return empty object. + // List "dir/" should return "dir/". let mut obs = op.lister(&dir).await?; let mut objects = HashMap::new(); while let Some(de) = obs.try_next().await? { objects.insert(de.path().to_string(), de); } - assert_eq!(objects.len(), 0, "dir should only return empty"); + assert_eq!( + objects.len(), + 1, + "only return the dir itself, but found: {objects:?}" + ); + assert_eq!( + objects[&dir].metadata().mode(), + EntryMode::DIR, + "given dir should exist and must be dir, but found: {objects:?}" + ); // List "dir" should return "dir/". let mut obs = op.lister(dir.trim_end_matches('/')).await?; @@ -278,15 +285,24 @@ pub async fn test_list_empty_dir(op: Operator) -> Result<()> { "given dir should exist and must be dir, but found: {objects:?}" ); - // List "dir/" should return empty object. + // List "dir/" recursively should return "dir/". let mut obs = op.lister_with(&dir).recursive(true).await?; let mut objects = HashMap::new(); while let Some(de) = obs.try_next().await? { objects.insert(de.path().to_string(), de); } - assert_eq!(objects.len(), 0, "dir should only return empty"); + assert_eq!( + objects.len(), + 1, + "only return the dir itself, but found: {objects:?}" + ); + assert_eq!( + objects[&dir].metadata().mode(), + EntryMode::DIR, + "given dir should exist and must be dir, but found: {objects:?}" + ); - // List "dir" should return "dir/". + // List "dir" recursively should return "dir/". let mut obs = op .lister_with(dir.trim_end_matches('/')) .recursive(true) @@ -353,23 +369,42 @@ pub async fn test_list_sub_dir(op: Operator) -> Result<()> { /// List dir should also to list nested dir. pub async fn test_list_nested_dir(op: Operator) -> Result<()> { let parent = format!("{}/", uuid::Uuid::new_v4()); + op.create_dir(&parent) + .await + .expect("create dir must succeed"); + let dir = format!("{parent}{}/", uuid::Uuid::new_v4()); + op.create_dir(&dir).await.expect("create must succeed"); let file_name = uuid::Uuid::new_v4().to_string(); let file_path = format!("{dir}{file_name}"); - let dir_name = format!("{}/", uuid::Uuid::new_v4()); - let dir_path = format!("{dir}{dir_name}"); - - op.create_dir(&dir).await.expect("create must succeed"); op.write(&file_path, "test_list_nested_dir") .await .expect("create must succeed"); + + let dir_name = format!("{}/", uuid::Uuid::new_v4()); + let dir_path = format!("{dir}{dir_name}"); op.create_dir(&dir_path).await.expect("create must succeed"); let obs = op.list(&parent).await?; - assert_eq!(obs.len(), 1, "parent should only got 1 entry"); - assert_eq!(obs[0].path(), dir); - assert_eq!(obs[0].metadata().mode(), EntryMode::DIR); + assert_eq!(obs.len(), 2, "parent should got 2 entry"); + let objects: HashMap<&str, &Entry> = obs.iter().map(|e| (e.path(), e)).collect(); + assert_eq!( + objects + .get(parent.as_str()) + .expect("parent should be found in list") + .metadata() + .mode(), + EntryMode::DIR + ); + assert_eq!( + objects + .get(dir.as_str()) + .expect("dir should be found in list") + .metadata() + .mode(), + EntryMode::DIR + ); let mut obs = op.lister(&dir).await?; let mut objects = HashMap::new(); @@ -379,7 +414,7 @@ pub async fn test_list_nested_dir(op: Operator) -> Result<()> { } debug!("got objects: {:?}", objects); - assert_eq!(objects.len(), 2, "dir should only got 2 objects"); + assert_eq!(objects.len(), 3, "dir should only got 3 objects"); // Check file let meta = op @@ -456,8 +491,10 @@ pub async fn test_list_with_start_after(op: Operator) -> Result<()> { let mut objects = op.lister_with(dir).start_after(&given[2]).await?; let mut actual = vec![]; while let Some(o) = objects.try_next().await? { - let path = o.path().to_string(); - actual.push(path) + if o.path() != dir { + let path = o.path().to_string(); + actual.push(path) + } } let expected: Vec = given.into_iter().skip(3).collect(); @@ -470,6 +507,8 @@ pub async fn test_list_with_start_after(op: Operator) -> Result<()> { } pub async fn test_list_root_with_recursive(op: Operator) -> Result<()> { + op.create_dir("/").await?; + let w = op.lister_with("").recursive(true).await?; let actual = w .try_collect::>() @@ -478,7 +517,7 @@ pub async fn test_list_root_with_recursive(op: Operator) -> Result<()> { .map(|v| v.path().to_string()) .collect::>(); - assert!(!actual.contains("/"), "empty root should not return itself"); + assert!(actual.contains("/"), "empty root should return itself"); assert!(!actual.contains(""), "empty root should not return empty"); Ok(()) } @@ -515,7 +554,7 @@ pub async fn test_list_dir_with_recursive(op: Operator) -> Result<()> { actual.sort(); let expected = vec![ - "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy", + "x/", "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy", ]; assert_eq!(actual, expected); Ok(()) @@ -585,7 +624,7 @@ pub async fn test_list_file_with_recursive(op: Operator) -> Result<()> { .collect::>(); actual.sort(); - let expected = vec!["yy"]; + let expected = vec!["y", "yy"]; assert_eq!(actual, expected); Ok(()) } diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs index c6bbd25f60b9..dd0647090efc 100644 --- a/core/tests/behavior/blocking_list.rs +++ b/core/tests/behavior/blocking_list.rs @@ -235,7 +235,7 @@ pub fn test_blocking_list_dir_with_recursive(op: BlockingOperator) -> Result<()> .collect::>(); actual.sort(); let expected = vec![ - "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy", + "x/", "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy", ]; assert_eq!(actual, expected); Ok(()) @@ -306,7 +306,7 @@ pub fn test_blocking_list_file_with_recursive(op: BlockingOperator) -> Result<() }) .collect::>(); actual.sort(); - let expected = vec!["yy"]; + let expected = vec!["y", "yy"]; assert_eq!(actual, expected); Ok(()) } diff --git a/integrations/dav-server/src/dir.rs b/integrations/dav-server/src/dir.rs index 10755f349821..4fbe36857806 100644 --- a/integrations/dav-server/src/dir.rs +++ b/integrations/dav-server/src/dir.rs @@ -15,28 +15,33 @@ // specific language governing permissions and limitations // under the License. +use super::metadata::OpendalMetaData; +use super::utils::*; use dav_server::fs::{DavDirEntry, DavMetaData, FsResult}; use futures::StreamExt; use futures::{FutureExt, Stream}; +use opendal::raw::normalize_path; use opendal::Operator; use opendal::{Entry, Lister}; use std::pin::Pin; use std::task::Poll::Ready; use std::task::{ready, Context, Poll}; -use super::metadata::OpendalMetaData; -use super::utils::*; - /// OpendalStream is a stream of `DavDirEntry` that is used to list the contents of a directory. pub struct OpendalStream { op: Operator, lister: Lister, + path: String, } impl OpendalStream { /// Create a new opendal stream. - pub fn new(op: Operator, lister: Lister) -> Self { - OpendalStream { op, lister } + pub fn new(op: Operator, lister: Lister, p: &str) -> Self { + OpendalStream { + op, + lister, + path: normalize_path(p), + } } } @@ -45,13 +50,18 @@ impl Stream for OpendalStream { fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let dav_stream = self.get_mut(); - match ready!(dav_stream.lister.poll_next_unpin(cx)) { - Some(entry) => { - let entry = entry.map_err(convert_error)?; - let webdav_entry = OpendalDirEntry::new(dav_stream.op.clone(), entry); - Ready(Some(Ok(Box::new(webdav_entry) as Box))) + loop { + match ready!(dav_stream.lister.poll_next_unpin(cx)) { + Some(entry) => { + let entry = entry.map_err(convert_error)?; + if entry.path() == dav_stream.path { + continue; + } + let webdav_entry = OpendalDirEntry::new(dav_stream.op.clone(), entry); + return Ready(Some(Ok(Box::new(webdav_entry) as Box))); + } + None => return Ready(None), } - None => Ready(None), } } } diff --git a/integrations/dav-server/src/fs.rs b/integrations/dav-server/src/fs.rs index fda9575c79fc..a1096bf47ad9 100644 --- a/integrations/dav-server/src/fs.rs +++ b/integrations/dav-server/src/fs.rs @@ -87,10 +87,11 @@ impl DavFileSystem for OpendalFs { _meta: ReadDirMeta, ) -> FsFuture>> { async move { + let path = path.as_url_string(); self.op - .lister(path.as_url_string().as_str()) + .lister(path.as_str()) .await - .map(|lister| OpendalStream::new(self.op.clone(), lister).boxed()) + .map(|lister| OpendalStream::new(self.op.clone(), lister, path.as_str()).boxed()) .map_err(convert_error) } .boxed() diff --git a/integrations/fuse3/src/file_system.rs b/integrations/fuse3/src/file_system.rs index 17b8cb442858..a3380d69ce48 100644 --- a/integrations/fuse3/src/file_system.rs +++ b/integrations/fuse3/src/file_system.rs @@ -29,6 +29,7 @@ use fuse3::Result; use futures_util::stream; use futures_util::stream::BoxStream; use futures_util::StreamExt; +use opendal::raw::normalize_path; use opendal::EntryMode; use opendal::ErrorKind; use opendal::Metadata; @@ -575,11 +576,21 @@ impl PathFilesystem for Filesystem { let mut current_dir = PathBuf::from(path); current_dir.push(""); // ref https://users.rust-lang.org/t/trailing-in-paths/43166 + let path = current_dir.to_string_lossy().to_string(); let children = self .op .lister(¤t_dir.to_string_lossy()) .await .map_err(opendal_error2errno)? + .filter_map(move |entry| { + let dir = normalize_path(path.as_str()); + async move { + match entry { + Ok(e) if e.path() == dir => None, + _ => Some(entry), + } + } + }) .enumerate() .map(|(i, entry)| { entry @@ -697,12 +708,22 @@ impl PathFilesystem for Filesystem { let uid = self.uid; let gid = self.gid; + let path = current_dir.to_string_lossy().to_string(); let children = self .op - .lister_with(¤t_dir.to_string_lossy()) + .lister_with(&path) .metakey(Metakey::ContentLength | Metakey::LastModified | Metakey::Mode) .await .map_err(opendal_error2errno)? + .filter_map(move |entry| { + let dir = normalize_path(path.as_str()); + async move { + match entry { + Ok(e) if e.path() == dir => None, + _ => Some(entry), + } + } + }) .enumerate() .map(move |(i, entry)| { entry