Merge branch 'apache:main' into main

.github/scripts/test_behavior/plan.py

@@ -86,7 +86,7 @@ class Hint:
     # Is bin ofs affected?
     bin_ofs: bool = field(default=False, init=False)
 
-    # Should we run all services test?
+    # Should we run all services tests?
     all_service: bool = field(default=False, init=False)
     # affected services set.
     services: set = field(default_factory=set, init=False)
@@ -95,7 +95,7 @@ class Hint:
 def calculate_hint(changed_files: list[str]) -> Hint:
     hint = Hint()
 
-    # Remove all files that ends with `.md`
+    # Remove all files that end with `.md`
     changed_files = [f for f in changed_files if not f.endswith(".md")]
 
     for p in changed_files:
@@ -179,9 +179,9 @@ def calculate_hint(changed_files: list[str]) -> Hint:
     return hint
 
 
-# unique_cases is used to only one setup for each service.
+# `unique_cases` is used to only one setup for each service.
 #
-# We need this because we have multiple setup for each service and they have already been
+# We need this because we have multiple setups for each service, and they have already been
 # tested by `core` workflow. So we can only test unique setup for each service for bindings.
 #
 # We make sure that we return the first setup for each service in alphabet order.
@@ -221,7 +221,13 @@ def generate_language_binding_cases(
 ) -> list[dict[str, str]]:
     cases = unique_cases(cases)
 
-    # Remove hdfs cases for java.
+    # Disable aliyun_drive case for every language.
+    #
+    # This is because aliyun_drive has a speed limit and tests may not be stable enough.
+    # Bindings may be treated as parallel requests, so we need to disable it for all languages.
+    cases = [v for v in cases if v["service"] != "aliyun_drive"]
+
+    # Remove hdfs cases for jav:a.
     if language == "java":
         cases = [v for v in cases if v["service"] != "hdfs"]
 
@@ -246,7 +252,7 @@ def generate_bin_cases(
     # Return empty if this bin is False
     if not getattr(hint, f"bin_{bin}"):
         return []
-    
+
     cases = unique_cases(cases)
 
     if bin == "ofs":

README.md

@@ -64,8 +64,8 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e
 
 ## For *ANY* methods
 
-| Name  | Description                                                        | Release                   | 
-|-------|--------------------------------------------------------------------|---------------------------| 
+| Name  | Description                                                        | Release                   |
+|-------|--------------------------------------------------------------------|---------------------------|
 | [oay] | Access data via API Gateway                                        | [![oay image]][oay crate] |
 | [oli] | Access data via Command Line (alternative to s3cmd, s3cli, azcopy) | [![oli image]][oli crate] |
 | [ofs] | Access data via POSIX file system API (alternative to s3fs)        | [![ofs image]][ofs crate] |
@@ -88,6 +88,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e
 | [object_store_opendal] | an [object_store] implementation using opendal.      | [![object_store image]][object_store crate] | [![Docs Release]][object_store release docs] [![Docs Dev]][object_store dev docs] |
 | [fuse3_opendal]        | Access data via integrations to [fuse3]              | -                                           | -                                                                                 |
 | [virtiofs_opendal]     | Access data via integrations to [vhost-user-backend] | -                                           | -                                                                                 |
+| [unftp-sbe-opendal]    | an [unftp] storage backend implementation using opendal. | -                                           | -                                                                                 |
 
 [dav-server-opendalfs]: integrations/dav-server/README.md
 [dav-server-rs]: https://github.com/messense/dav-server-rs
@@ -109,10 +110,13 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e
 [virtiofs_opendal]: integrations/virtiofs/README.md
 [vhost-user-backend]: https://docs.rs/vhost-user-backend
 
+[unftp-sbe-opendal]: integrations/unftp-sbe/README.md
+[unftp]: https://crates.io/crates/unftp
+
 ## For *ANY* services
 
-| Type                           | Services                                                                                                                                 | 
-|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| 
+| Type                           | Services                                                                                                                                 |
+|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|
 | Standard Storage Protocols     | ftp http [sftp] [webdav]                                                                                                                 |
 | Object Storage Services        | [azblob] [cos] [gcs] [obs] [oss] [s3] <br> [b2] [openstack_swift] [upyun] [vercel_blob]                                                  |
 | File Storage Services          | fs [alluxio] [azdls] [azfile] [chainsafe] [compfs] <br> [dbfs] [gridfs] [hdfs] [hdfs_native] [ipfs] [webhdfs]                            |

core/Cargo.toml

@@ -344,11 +344,7 @@ hdfs-native = { version = "0.9.4", optional = true }
 # for services-surrealdb
 surrealdb = { version = "1.3.0", optional = true, features = ["protocol-http"] }
 # for services-compfs
-compio = { version = "0.10.0", optional = true, features = [
-  "runtime",
-  "bytes",
-  "polling",
-] }
+compio = { version = "0.11.0", optional = true, features = ["runtime", "bytes", "polling", "dispatcher"] }
 # for services-s3
 crc32c = { version = "0.6.6", optional = true }
 

core/src/services/aliyun_drive/backend.rs

@@ -43,31 +43,31 @@ use crate::*;
 #[serde(default)]
 #[non_exhaustive]
 pub struct AliyunDriveConfig {
-    /// root of this backend.
+    /// The Root of this backend.
     ///
     /// All operations will happen under this root.
     ///
-    /// default to `/` if not set.
+    /// Default to `/` if not set.
     pub root: Option<String>,
-    /// access_token of this backend.
+    /// The access_token of this backend.
     ///
     /// Solution for client-only purpose. #4733
     ///
-    /// required if no client_id, client_secret and refresh_token are provided.
+    /// Required if no client_id, client_secret and refresh_token are provided.
     pub access_token: Option<String>,
-    /// client_id of this backend.
+    /// The client_id of this backend.
     ///
-    /// required if no access_token is provided.
+    /// Required if no access_token is provided.
     pub client_id: Option<String>,
-    /// client_secret of this backend.
+    /// The client_secret of this backend.
     ///
-    /// required if no access_token is provided.
+    /// Required if no access_token is provided.
     pub client_secret: Option<String>,
-    /// refresh_token of this backend.
+    /// The refresh_token of this backend.
     ///
-    /// required if no access_token is provided.
+    /// Required if no access_token is provided.
     pub refresh_token: Option<String>,
-    /// drive_type of this backend.
+    /// The drive_type of this backend.
     ///
     /// All operations will happen under this type of drive.
     ///
@@ -106,7 +106,7 @@ impl Debug for AliyunDriveBuilder {
 }
 
 impl AliyunDriveBuilder {
-    /// Set root of this backend.
+    /// Set the root of this backend.
     ///
     /// All operations will happen under this root.
     pub fn root(&mut self, root: &str) -> &mut Self {
@@ -342,8 +342,8 @@ impl Access for AliyunDriveBackend {
         let parent_path = get_parent(to);
         let parent_file_id = self.core.ensure_dir_exists(parent_path).await?;
 
-        // if from and to are going to be placed in the same folder
-        // copy_path will fail as we cannot change name during this action.
+        // if from and to are going to be placed in the same folder,
+        // copy_path will fail as we cannot change the name during this action.
         // it has to be auto renamed.
         let auto_rename = file.parent_file_id == parent_file_id;
         let res = self
@@ -445,8 +445,10 @@ impl Access for AliyunDriveBackend {
                 let file: AliyunDriveFile =
                     serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
                 Some(AliyunDriveParent {
-                    parent_path: path.to_string(),
-                    parent_file_id: file.file_id,
+                    file_id: file.file_id,
+                    name: file.name,
+                    path: path.to_string(),
+                    updated_at: file.updated_at,
                 })
             }
         };

core/src/services/aliyun_drive/core.rs

@@ -40,11 +40,11 @@ pub enum DriveType {
     Default,
     /// Use the backup type of Aliyun Drive.
     ///
-    /// fallback to the default type if no backup drive found.
+    /// Fallback to the default type if no backup drive is found.
     Backup,
     /// Use the resource type of Aliyun Drive.
     ///
-    /// fallback to the default type if no resource drive found.
+    /// Fallback to the default type if no resource drive is found.
     Resource,
 }
 

core/src/services/aliyun_drive/lister.rs

@@ -38,8 +38,10 @@ pub struct AliyunDriveLister {
 }
 
 pub struct AliyunDriveParent {
-    pub parent_path: String,
-    pub parent_file_id: String,
+    pub file_id: String,
+    pub name: String,
+    pub path: String,
+    pub updated_at: String,
 }
 
 impl AliyunDriveLister {
@@ -64,15 +66,32 @@ impl oio::PageList for AliyunDriveLister {
         };
 
         let offset = if ctx.token.is_empty() {
+            if !parent.path.ends_with('/') {
+                // List "dir" should contains "dir/".
+                let path = if !parent.path.starts_with('/') {
+                    format!("/{}", parent.path)
+                } else {
+                    parent.path.clone()
+                };
+                ctx.entries.push_back(Entry::new(
+                    &format!("{}/", path),
+                    Metadata::new(EntryMode::DIR).with_last_modified(
+                        parent
+                            .updated_at
+                            .parse::<chrono::DateTime<Utc>>()
+                            .map_err(|e| {
+                                Error::new(ErrorKind::Unexpected, "parse last modified time")
+                                    .set_source(e)
+                            })?,
+                    ),
+                ));
+            }
             None
         } else {
             Some(ctx.token.clone())
         };
 
-        let res = self
-            .core
-            .list(&parent.parent_file_id, self.limit, offset)
-            .await;
+        let res = self.core.list(&parent.file_id, self.limit, offset).await;
         let res = match res {
             Err(err) if err.kind() == ErrorKind::NotFound => {
                 ctx.done = true;
@@ -92,10 +111,10 @@ impl oio::PageList for AliyunDriveLister {
         let n = result.items.len();
 
         for item in result.items {
-            let path = if parent.parent_path.starts_with('/') {
-                build_abs_path(&parent.parent_path, &item.name)
+            let path = if parent.path.starts_with('/') {
+                build_abs_path(&parent.path, &item.name)
             } else {
-                build_abs_path(&format!("/{}", &parent.parent_path), &item.name)
+                build_abs_path(&format!("/{}", &parent.path), &item.name)
             };
 
             let (path, md) = if item.path_type == "folder" {