From 04e96519ebc9451d73b2ae0965555edf303dac3b Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Mon, 18 Sep 2023 20:47:33 +0800 Subject: [PATCH 1/9] port `WebdavFs` to dav-server-fs-opendal --- integrations/README.md | 2 + integrations/dav-server-fs-opendal/.gitignore | 1 + integrations/dav-server-fs-opendal/Cargo.toml | 58 +++++ integrations/dav-server-fs-opendal/README.md | 3 + integrations/dav-server-fs-opendal/src/lib.rs | 6 + .../src/webdav_dir_entry.rs | 54 +++++ .../dav-server-fs-opendal/src/webdav_file.rs | 94 ++++++++ .../src/webdav_metadata.rs | 62 +++++ .../dav-server-fs-opendal/src/webdavfs.rs | 220 ++++++++++++++++++ .../dav-server-fs-opendal/tests/test.rs | 38 +++ 10 files changed, 538 insertions(+) create mode 100644 integrations/dav-server-fs-opendal/.gitignore create mode 100644 integrations/dav-server-fs-opendal/Cargo.toml create mode 100644 integrations/dav-server-fs-opendal/README.md create mode 100644 integrations/dav-server-fs-opendal/src/lib.rs create mode 100644 integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs create mode 100644 integrations/dav-server-fs-opendal/src/webdav_file.rs create mode 100644 integrations/dav-server-fs-opendal/src/webdav_metadata.rs create mode 100644 integrations/dav-server-fs-opendal/src/webdavfs.rs create mode 100644 integrations/dav-server-fs-opendal/tests/test.rs diff --git a/integrations/README.md b/integrations/README.md index dd89e76cf124..23724f547b39 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,3 +5,5 @@ This folder contains the integrations for OpenDAL. Integrations are used to inte ## Available Integrations - [`object_store_opendal`](./object_store): Use OpenDAL as a backend for the [object_store](https://docs.rs/object_store/latest/object_store/). + +- [`dav-server-fs-opendal`](./dav-server-fs-opendal/): Use OpenDAL as a backend to access data in various service with WebDAV protocol. diff --git a/integrations/dav-server-fs-opendal/.gitignore b/integrations/dav-server-fs-opendal/.gitignore new file mode 100644 index 000000000000..03314f77b5aa --- /dev/null +++ b/integrations/dav-server-fs-opendal/.gitignore @@ -0,0 +1 @@ +Cargo.lock diff --git a/integrations/dav-server-fs-opendal/Cargo.toml b/integrations/dav-server-fs-opendal/Cargo.toml new file mode 100644 index 000000000000..0c7a847f0376 --- /dev/null +++ b/integrations/dav-server-fs-opendal/Cargo.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +description = "Use OpenDAL as a backend to access data in various service with WebDAV protocol" +name = "dav-server-fs-opendal" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +anyhow = "1" +axum = "0.6" +chrono = "0.4.28" +clap = { version = "4", features = ["cargo", "string"] } +dirs = "5.0.0" +# bytes = { version = "1.4.0", optional = true } +# dav-server = { version = "0.5.5", optional = true } +# futures-util = { version = "0.3.16", optional = true } +bytes = { version = "1.4.0"} +dav-server = { version = "0.5.5"} +futures-util = { version = "0.3.16"} +futures = "0.3" +opendal.workspace = true +quick-xml = { version = "0.29", features = ["serialize", "overlapped-lists"] } +serde = { version = "1", features = ["derive"] } +tokio = { version = "1.27", features = [ + "fs", + "macros", + "rt-multi-thread", + "io-std", +] } +toml = "0.7.3" +tower = "0.4" +tower-http = { version = "0.4", features = ["trace"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +url = "2.3.1" +uuid = { version = "1", features = ["v4", "fast-rng"] } diff --git a/integrations/dav-server-fs-opendal/README.md b/integrations/dav-server-fs-opendal/README.md new file mode 100644 index 000000000000..907cae3731d7 --- /dev/null +++ b/integrations/dav-server-fs-opendal/README.md @@ -0,0 +1,3 @@ +# dav-server-fs-opendal + +todo diff --git a/integrations/dav-server-fs-opendal/src/lib.rs b/integrations/dav-server-fs-opendal/src/lib.rs new file mode 100644 index 000000000000..7704978fa6d9 --- /dev/null +++ b/integrations/dav-server-fs-opendal/src/lib.rs @@ -0,0 +1,6 @@ +mod webdav_dir_entry; +mod webdav_file; +mod webdav_metadata; +pub mod webdavfs; + +pub use dav_server::*; diff --git a/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs b/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs new file mode 100644 index 000000000000..c3836db3ef48 --- /dev/null +++ b/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavDirEntry; +use futures::FutureExt; +use opendal::Entry; +use opendal::Operator; + +use super::webdav_file::convert_error; +use super::webdav_metadata::WebdavMetaData; + +pub struct WebDAVDirEntry { + dir_entry: Entry, + op: Operator, +} + +impl DavDirEntry for WebDAVDirEntry { + fn name(&self) -> Vec { + self.dir_entry.name().as_bytes().to_vec() + } + + fn metadata(&self) -> dav_server::fs::FsFuture> { + async move { + self.op + .stat(self.dir_entry.path()) + .await + .map(|metadata| { + Box::new(WebdavMetaData::new(metadata)) as Box + }) + .map_err(convert_error) + } + .boxed() + } +} + +impl WebDAVDirEntry { + pub fn new(dir_entry: Entry, op: Operator) -> Self { + WebDAVDirEntry { dir_entry, op } + } +} diff --git a/integrations/dav-server-fs-opendal/src/webdav_file.rs b/integrations/dav-server-fs-opendal/src/webdav_file.rs new file mode 100644 index 000000000000..53e99c07fceb --- /dev/null +++ b/integrations/dav-server-fs-opendal/src/webdav_file.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::io::SeekFrom; + +use bytes::Bytes; +use dav_server::davpath::DavPath; +use dav_server::fs::DavFile; +use dav_server::fs::DavMetaData; +use dav_server::fs::FsFuture; +use dav_server::fs::OpenOptions; +use futures::FutureExt; +use opendal::Operator; + +use super::webdav_metadata::WebdavMetaData; + +#[derive(Debug)] +pub struct WebdavFile { + pub op: Operator, + pub path: DavPath, + pub options: OpenOptions, +} + +impl DavFile for WebdavFile { + fn read_bytes(&mut self, count: usize) -> FsFuture { + async move { + let file_path = self.path.as_url_string(); + self.op + .read_with(&file_path) + .range(0..count as u64) + .await + .map(Bytes::from) + .map_err(convert_error) + } + .boxed() + } + + fn metadata(&mut self) -> FsFuture> { + async move { + self.op + .stat(self.path.as_url_string().as_str()) + .await + .map(|opendal_metadata| { + Box::new(WebdavMetaData::new(opendal_metadata)) as Box + }) + .map_err(convert_error) + } + .boxed() + } + + fn write_buf(&mut self, buf: Box) -> FsFuture<()> { + self.write_bytes(Bytes::copy_from_slice(buf.chunk())) + } + + fn write_bytes(&mut self, buf: Bytes) -> FsFuture<()> { + async move { + let file_path = self.path.as_url_string(); + self.op.write(&file_path, buf).await.map_err(convert_error) + } + .boxed() + } + + fn seek(&mut self, _pos: SeekFrom) -> FsFuture { + futures_util::future::err(dav_server::fs::FsError::NotImplemented).boxed() + } + + fn flush(&mut self) -> FsFuture<()> { + futures_util::future::ok(()).boxed() + } +} + +pub fn convert_error(opendal_error: opendal::Error) -> dav_server::fs::FsError { + match opendal_error.kind() { + opendal::ErrorKind::AlreadyExists | opendal::ErrorKind::IsSameFile => { + dav_server::fs::FsError::Exists + } + opendal::ErrorKind::NotFound => dav_server::fs::FsError::NotFound, + _ => dav_server::fs::FsError::GeneralFailure, + } +} diff --git a/integrations/dav-server-fs-opendal/src/webdav_metadata.rs b/integrations/dav-server-fs-opendal/src/webdav_metadata.rs new file mode 100644 index 000000000000..03295125d5ee --- /dev/null +++ b/integrations/dav-server-fs-opendal/src/webdav_metadata.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavMetaData; +use dav_server::fs::FsError; +use opendal::Metadata; + +#[derive(Debug, Clone)] +pub struct WebdavMetaData { + metadata: Metadata, +} + +impl WebdavMetaData { + pub fn new(metadata: Metadata) -> Self { + WebdavMetaData { metadata } + } +} + +impl DavMetaData for WebdavMetaData { + fn len(&self) -> u64 { + self.metadata.content_length() + } + + fn modified(&self) -> dav_server::fs::FsResult { + match self.metadata.last_modified() { + Some(t) => Ok(t.into()), + None => Err(FsError::GeneralFailure), + } + } + + fn is_dir(&self) -> bool { + self.metadata.is_dir() + } + + fn is_file(&self) -> bool { + self.metadata.is_file() + } + + fn etag(&self) -> Option { + self.metadata.etag().map(|s| s.to_string()) + } + + fn status_changed(&self) -> dav_server::fs::FsResult { + self.metadata + .last_modified() + .map_or(Err(FsError::GeneralFailure), |t| Ok(t.into())) + } +} diff --git a/integrations/dav-server-fs-opendal/src/webdavfs.rs b/integrations/dav-server-fs-opendal/src/webdavfs.rs new file mode 100644 index 000000000000..392926c307a6 --- /dev/null +++ b/integrations/dav-server-fs-opendal/src/webdavfs.rs @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::path::Path; +use std::pin::Pin; +use std::task::Poll::Pending; +use std::task::Poll::Ready; + +use dav_server::davpath::DavPath; +use dav_server::fs::DavDirEntry; +use dav_server::fs::DavFile; +use dav_server::fs::DavFileSystem; +use dav_server::fs::DavMetaData; +use dav_server::fs::FsError; +use futures::FutureExt; +use futures_util::Stream; +use futures_util::StreamExt; +use opendal::Lister; +use opendal::Operator; + +use super::webdav_file::convert_error; +use super::webdav_file::WebdavFile; +use super::webdav_metadata::WebdavMetaData; +use crate::webdav_dir_entry::WebDAVDirEntry; + +#[derive(Clone)] +pub struct WebdavFs { + pub op: Operator, +} + +impl DavFileSystem for WebdavFs { + fn open<'a>( + &'a self, + path: &'a dav_server::davpath::DavPath, + options: dav_server::fs::OpenOptions, + ) -> dav_server::fs::FsFuture> { + async move { + let file = WebdavFile { + op: self.op.clone(), + path: path.clone(), + options, + }; + Ok(Box::new(file) as Box) + } + .boxed() + } + + fn read_dir<'a>( + &'a self, + path: &'a dav_server::davpath::DavPath, + _meta: dav_server::fs::ReadDirMeta, + ) -> dav_server::fs::FsFuture>> + { + async move { + self.op + .lister(path.as_url_string().as_str()) + .await + .map(|lister| DavStream::new(self.op.clone(), lister).boxed()) + .map_err(convert_error) + } + .boxed() + } + + fn metadata<'a>( + &'a self, + path: &'a dav_server::davpath::DavPath, + ) -> dav_server::fs::FsFuture> { + async move { + let opendal_metadata = self.op.stat(path.as_url_string().as_str()).await; + match opendal_metadata { + Ok(metadata) => { + let webdav_metadata = WebdavMetaData::new(metadata); + Ok(Box::new(webdav_metadata) as Box) + } + Err(e) => Err(convert_error(e)), + } + } + .boxed() + } + + fn create_dir<'a>(&'a self, path: &'a DavPath) -> dav_server::fs::FsFuture<()> { + async move { + let path = path.as_url_string(); + + // check if the parent path is exist. + // During MKCOL processing, a server MUST make the Request-URI a member of its parent collection, unless the Request-URI is "/". If no such ancestor exists, the method MUST fail. + // refer to https://datatracker.ietf.org/doc/html/rfc2518#section-8.3.1 + let parent = Path::new(&path).parent().unwrap(); + match self.op.is_exist(parent.to_str().unwrap()).await { + Ok(exist) => { + if !exist && parent != Path::new("/") { + return Err(dav_server::fs::FsError::NotFound); + } + } + Err(e) => { + return Err(convert_error(e)); + } + } + + let path = path.as_str(); + // check if the given path is exist (MKCOL on existing collection should fail (RFC2518:8.3.1)) + let exist = self.op.is_exist(path).await; + match exist { + Ok(exist) => match exist { + true => Err(dav_server::fs::FsError::Exists), + false => { + let res = self.op.create_dir(path).await; + match res { + Ok(_) => Ok(()), + Err(e) => Err(convert_error(e)), + } + } + }, + Err(e) => Err(convert_error(e)), + } + } + .boxed() + } + + fn remove_file<'a>(&'a self, path: &'a DavPath) -> dav_server::fs::FsFuture<()> { + async move { + self.op + .delete(path.as_url_string().as_str()) + .await + .map_err(convert_error) + } + .boxed() + } + + fn remove_dir<'a>(&'a self, path: &'a DavPath) -> dav_server::fs::FsFuture<()> { + self.remove_file(path) + } + + fn copy<'a>(&'a self, from: &'a DavPath, to: &'a DavPath) -> dav_server::fs::FsFuture<()> { + async move { + let from_path = from + .as_rel_ospath() + .to_str() + .ok_or(FsError::GeneralFailure)?; + let to_path = to.as_rel_ospath().to_str().ok_or(FsError::GeneralFailure)?; + self.op + .copy(from_path, to_path) + .await + .map_err(convert_error) + } + .boxed() + } + + fn rename<'a>(&'a self, from: &'a DavPath, to: &'a DavPath) -> dav_server::fs::FsFuture<()> { + async move { + let from_path = from + .as_rel_ospath() + .to_str() + .ok_or(FsError::GeneralFailure)?; + let to_path = to.as_rel_ospath().to_str().ok_or(FsError::GeneralFailure)?; + if from.is_collection() { + let _ = self.remove_file(to).await; + } + self.op + .rename(from_path, to_path) + .await + .map_err(convert_error) + } + .boxed() + } +} + +impl WebdavFs { + pub fn new(op: Operator) -> Box { + Box::new(WebdavFs { op }) + } +} + +struct DavStream { + op: Operator, + lister: Lister, +} + +impl Stream for DavStream { + type Item = Box; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let dav_stream = self.get_mut(); + let lister = Pin::new(&mut dav_stream.lister).get_mut(); + + match Pin::new(lister).poll_next(cx) { + Ready(entry) => match entry { + Some(entry) => { + let webdav_entry = WebDAVDirEntry::new(entry.unwrap(), dav_stream.op.clone()); + Ready(Some(Box::new(webdav_entry) as Box)) + } + None => Ready(None), + }, + Pending => Pending, + } + } +} + +impl DavStream { + fn new(op: Operator, lister: Lister) -> Self { + DavStream { op, lister } + } +} diff --git a/integrations/dav-server-fs-opendal/tests/test.rs b/integrations/dav-server-fs-opendal/tests/test.rs new file mode 100644 index 000000000000..93a62f144fce --- /dev/null +++ b/integrations/dav-server-fs-opendal/tests/test.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use dav_server_fs_opendal::webdavfs::WebdavFs; +use dav_server_fs_opendal::davpath::DavPath; +use dav_server_fs_opendal::fs::DavFileSystem; +use opendal::services::Fs; +use opendal::Operator; + +#[tokio::test] +async fn test() -> Result<()> { + let mut builder = Fs::default(); + builder.root("/tmp"); + + let op = Operator::new(builder)?.finish(); + + let webdavfs = WebdavFs::new(op); + + let metadata = webdavfs.metadata(&DavPath::new("/").unwrap()).await.unwrap(); + println!("{}", metadata.is_dir()); + + Ok(()) +} From 6dd687ed6b3e4535b54bbb065713deb74e7bad75 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Mon, 18 Sep 2023 20:56:37 +0800 Subject: [PATCH 2/9] update --- integrations/dav-server-fs-opendal/src/lib.rs | 17 +++++++++++++++++ .../dav-server-fs-opendal/tests/test.rs | 7 +++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/integrations/dav-server-fs-opendal/src/lib.rs b/integrations/dav-server-fs-opendal/src/lib.rs index 7704978fa6d9..fff369a5427c 100644 --- a/integrations/dav-server-fs-opendal/src/lib.rs +++ b/integrations/dav-server-fs-opendal/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod webdav_dir_entry; mod webdav_file; mod webdav_metadata; diff --git a/integrations/dav-server-fs-opendal/tests/test.rs b/integrations/dav-server-fs-opendal/tests/test.rs index 93a62f144fce..ece739514e0e 100644 --- a/integrations/dav-server-fs-opendal/tests/test.rs +++ b/integrations/dav-server-fs-opendal/tests/test.rs @@ -16,9 +16,9 @@ // under the License. use anyhow::Result; -use dav_server_fs_opendal::webdavfs::WebdavFs; use dav_server_fs_opendal::davpath::DavPath; use dav_server_fs_opendal::fs::DavFileSystem; +use dav_server_fs_opendal::webdavfs::WebdavFs; use opendal::services::Fs; use opendal::Operator; @@ -31,7 +31,10 @@ async fn test() -> Result<()> { let webdavfs = WebdavFs::new(op); - let metadata = webdavfs.metadata(&DavPath::new("/").unwrap()).await.unwrap(); + let metadata = webdavfs + .metadata(&DavPath::new("/").unwrap()) + .await + .unwrap(); println!("{}", metadata.is_dir()); Ok(()) From eaae08978d5d0c684613bddf1a95ca8e58ac1f2c Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Tue, 19 Sep 2023 21:27:26 +0800 Subject: [PATCH 3/9] update --- integrations/README.md | 2 +- integrations/dav-server-fs-opendal/.gitignore | 1 - integrations/dav-server-fs-opendal/README.md | 3 - .../src/webdav_dir_entry.rs | 54 ----------- .../dav-server-fs-opendal/src/webdav_file.rs | 94 ------------------- .../src/webdav_metadata.rs | 62 ------------ integrations/dav-server/.gitignore | 0 .../Cargo.toml | 21 +---- integrations/dav-server/README.md | 3 + .../src/lib.rs | 2 +- .../dav-server/src}/webdav_dir_entry.rs | 0 .../dav-server/src}/webdav_file.rs | 12 ++- .../dav-server/src}/webdav_metadata.rs | 0 .../src/webdavfs.rs | 8 +- .../tests/test.rs | 6 +- 15 files changed, 23 insertions(+), 245 deletions(-) delete mode 100644 integrations/dav-server-fs-opendal/.gitignore delete mode 100644 integrations/dav-server-fs-opendal/README.md delete mode 100644 integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs delete mode 100644 integrations/dav-server-fs-opendal/src/webdav_file.rs delete mode 100644 integrations/dav-server-fs-opendal/src/webdav_metadata.rs create mode 100644 integrations/dav-server/.gitignore rename integrations/{dav-server-fs-opendal => dav-server}/Cargo.toml (70%) create mode 100644 integrations/dav-server/README.md rename integrations/{dav-server-fs-opendal => dav-server}/src/lib.rs (96%) rename {bin/oay/src/services/webdav => integrations/dav-server/src}/webdav_dir_entry.rs (100%) rename {bin/oay/src/services/webdav => integrations/dav-server/src}/webdav_file.rs (95%) rename {bin/oay/src/services/webdav => integrations/dav-server/src}/webdav_metadata.rs (100%) rename integrations/{dav-server-fs-opendal => dav-server}/src/webdavfs.rs (97%) rename integrations/{dav-server-fs-opendal => dav-server}/tests/test.rs (89%) diff --git a/integrations/README.md b/integrations/README.md index 23724f547b39..ba5dfb6b4a75 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -6,4 +6,4 @@ This folder contains the integrations for OpenDAL. Integrations are used to inte - [`object_store_opendal`](./object_store): Use OpenDAL as a backend for the [object_store](https://docs.rs/object_store/latest/object_store/). -- [`dav-server-fs-opendal`](./dav-server-fs-opendal/): Use OpenDAL as a backend to access data in various service with WebDAV protocol. +- [`dav-server-opendalfs`](./dav-server-opendalfs/): Use OpenDAL as a backend to access data in various service with WebDAV protocol. diff --git a/integrations/dav-server-fs-opendal/.gitignore b/integrations/dav-server-fs-opendal/.gitignore deleted file mode 100644 index 03314f77b5aa..000000000000 --- a/integrations/dav-server-fs-opendal/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Cargo.lock diff --git a/integrations/dav-server-fs-opendal/README.md b/integrations/dav-server-fs-opendal/README.md deleted file mode 100644 index 907cae3731d7..000000000000 --- a/integrations/dav-server-fs-opendal/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# dav-server-fs-opendal - -todo diff --git a/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs b/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs deleted file mode 100644 index c3836db3ef48..000000000000 --- a/integrations/dav-server-fs-opendal/src/webdav_dir_entry.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use dav_server::fs::DavDirEntry; -use futures::FutureExt; -use opendal::Entry; -use opendal::Operator; - -use super::webdav_file::convert_error; -use super::webdav_metadata::WebdavMetaData; - -pub struct WebDAVDirEntry { - dir_entry: Entry, - op: Operator, -} - -impl DavDirEntry for WebDAVDirEntry { - fn name(&self) -> Vec { - self.dir_entry.name().as_bytes().to_vec() - } - - fn metadata(&self) -> dav_server::fs::FsFuture> { - async move { - self.op - .stat(self.dir_entry.path()) - .await - .map(|metadata| { - Box::new(WebdavMetaData::new(metadata)) as Box - }) - .map_err(convert_error) - } - .boxed() - } -} - -impl WebDAVDirEntry { - pub fn new(dir_entry: Entry, op: Operator) -> Self { - WebDAVDirEntry { dir_entry, op } - } -} diff --git a/integrations/dav-server-fs-opendal/src/webdav_file.rs b/integrations/dav-server-fs-opendal/src/webdav_file.rs deleted file mode 100644 index 53e99c07fceb..000000000000 --- a/integrations/dav-server-fs-opendal/src/webdav_file.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::io::SeekFrom; - -use bytes::Bytes; -use dav_server::davpath::DavPath; -use dav_server::fs::DavFile; -use dav_server::fs::DavMetaData; -use dav_server::fs::FsFuture; -use dav_server::fs::OpenOptions; -use futures::FutureExt; -use opendal::Operator; - -use super::webdav_metadata::WebdavMetaData; - -#[derive(Debug)] -pub struct WebdavFile { - pub op: Operator, - pub path: DavPath, - pub options: OpenOptions, -} - -impl DavFile for WebdavFile { - fn read_bytes(&mut self, count: usize) -> FsFuture { - async move { - let file_path = self.path.as_url_string(); - self.op - .read_with(&file_path) - .range(0..count as u64) - .await - .map(Bytes::from) - .map_err(convert_error) - } - .boxed() - } - - fn metadata(&mut self) -> FsFuture> { - async move { - self.op - .stat(self.path.as_url_string().as_str()) - .await - .map(|opendal_metadata| { - Box::new(WebdavMetaData::new(opendal_metadata)) as Box - }) - .map_err(convert_error) - } - .boxed() - } - - fn write_buf(&mut self, buf: Box) -> FsFuture<()> { - self.write_bytes(Bytes::copy_from_slice(buf.chunk())) - } - - fn write_bytes(&mut self, buf: Bytes) -> FsFuture<()> { - async move { - let file_path = self.path.as_url_string(); - self.op.write(&file_path, buf).await.map_err(convert_error) - } - .boxed() - } - - fn seek(&mut self, _pos: SeekFrom) -> FsFuture { - futures_util::future::err(dav_server::fs::FsError::NotImplemented).boxed() - } - - fn flush(&mut self) -> FsFuture<()> { - futures_util::future::ok(()).boxed() - } -} - -pub fn convert_error(opendal_error: opendal::Error) -> dav_server::fs::FsError { - match opendal_error.kind() { - opendal::ErrorKind::AlreadyExists | opendal::ErrorKind::IsSameFile => { - dav_server::fs::FsError::Exists - } - opendal::ErrorKind::NotFound => dav_server::fs::FsError::NotFound, - _ => dav_server::fs::FsError::GeneralFailure, - } -} diff --git a/integrations/dav-server-fs-opendal/src/webdav_metadata.rs b/integrations/dav-server-fs-opendal/src/webdav_metadata.rs deleted file mode 100644 index 03295125d5ee..000000000000 --- a/integrations/dav-server-fs-opendal/src/webdav_metadata.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use dav_server::fs::DavMetaData; -use dav_server::fs::FsError; -use opendal::Metadata; - -#[derive(Debug, Clone)] -pub struct WebdavMetaData { - metadata: Metadata, -} - -impl WebdavMetaData { - pub fn new(metadata: Metadata) -> Self { - WebdavMetaData { metadata } - } -} - -impl DavMetaData for WebdavMetaData { - fn len(&self) -> u64 { - self.metadata.content_length() - } - - fn modified(&self) -> dav_server::fs::FsResult { - match self.metadata.last_modified() { - Some(t) => Ok(t.into()), - None => Err(FsError::GeneralFailure), - } - } - - fn is_dir(&self) -> bool { - self.metadata.is_dir() - } - - fn is_file(&self) -> bool { - self.metadata.is_file() - } - - fn etag(&self) -> Option { - self.metadata.etag().map(|s| s.to_string()) - } - - fn status_changed(&self) -> dav_server::fs::FsResult { - self.metadata - .last_modified() - .map_or(Err(FsError::GeneralFailure), |t| Ok(t.into())) - } -} diff --git a/integrations/dav-server/.gitignore b/integrations/dav-server/.gitignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/integrations/dav-server-fs-opendal/Cargo.toml b/integrations/dav-server/Cargo.toml similarity index 70% rename from integrations/dav-server-fs-opendal/Cargo.toml rename to integrations/dav-server/Cargo.toml index 0c7a847f0376..aa82c51a14f0 100644 --- a/integrations/dav-server-fs-opendal/Cargo.toml +++ b/integrations/dav-server/Cargo.toml @@ -17,7 +17,7 @@ [package] description = "Use OpenDAL as a backend to access data in various service with WebDAV protocol" -name = "dav-server-fs-opendal" +name = "dav-server-opendalfs" authors.workspace = true edition.workspace = true @@ -29,17 +29,12 @@ version.workspace = true [dependencies] anyhow = "1" -axum = "0.6" chrono = "0.4.28" -clap = { version = "4", features = ["cargo", "string"] } dirs = "5.0.0" -# bytes = { version = "1.4.0", optional = true } -# dav-server = { version = "0.5.5", optional = true } -# futures-util = { version = "0.3.16", optional = true } -bytes = { version = "1.4.0"} -dav-server = { version = "0.5.5"} -futures-util = { version = "0.3.16"} +bytes = { version = "1.4.0" } +dav-server = { version = "0.5.5" } futures = "0.3" +futures-util = { version = "0.3.16" } opendal.workspace = true quick-xml = { version = "0.29", features = ["serialize", "overlapped-lists"] } serde = { version = "1", features = ["derive"] } @@ -49,10 +44,4 @@ tokio = { version = "1.27", features = [ "rt-multi-thread", "io-std", ] } -toml = "0.7.3" -tower = "0.4" -tower-http = { version = "0.4", features = ["trace"] } -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -url = "2.3.1" -uuid = { version = "1", features = ["v4", "fast-rng"] } + diff --git a/integrations/dav-server/README.md b/integrations/dav-server/README.md new file mode 100644 index 000000000000..15420debac0d --- /dev/null +++ b/integrations/dav-server/README.md @@ -0,0 +1,3 @@ +# dav-server-opendalfs + +`dav-server-opendalfs` is a integration which use OpenDAL as a backend to access data in various service with WebDAV protocol. diff --git a/integrations/dav-server-fs-opendal/src/lib.rs b/integrations/dav-server/src/lib.rs similarity index 96% rename from integrations/dav-server-fs-opendal/src/lib.rs rename to integrations/dav-server/src/lib.rs index fff369a5427c..ca642b2fbeba 100644 --- a/integrations/dav-server-fs-opendal/src/lib.rs +++ b/integrations/dav-server/src/lib.rs @@ -20,4 +20,4 @@ mod webdav_file; mod webdav_metadata; pub mod webdavfs; -pub use dav_server::*; +pub use dav_server::DavHandler; diff --git a/bin/oay/src/services/webdav/webdav_dir_entry.rs b/integrations/dav-server/src/webdav_dir_entry.rs similarity index 100% rename from bin/oay/src/services/webdav/webdav_dir_entry.rs rename to integrations/dav-server/src/webdav_dir_entry.rs diff --git a/bin/oay/src/services/webdav/webdav_file.rs b/integrations/dav-server/src/webdav_file.rs similarity index 95% rename from bin/oay/src/services/webdav/webdav_file.rs rename to integrations/dav-server/src/webdav_file.rs index 53e99c07fceb..3406b1dc6d9c 100644 --- a/bin/oay/src/services/webdav/webdav_file.rs +++ b/integrations/dav-server/src/webdav_file.rs @@ -22,7 +22,6 @@ use dav_server::davpath::DavPath; use dav_server::fs::DavFile; use dav_server::fs::DavMetaData; use dav_server::fs::FsFuture; -use dav_server::fs::OpenOptions; use futures::FutureExt; use opendal::Operator; @@ -30,9 +29,14 @@ use super::webdav_metadata::WebdavMetaData; #[derive(Debug)] pub struct WebdavFile { - pub op: Operator, - pub path: DavPath, - pub options: OpenOptions, + op: Operator, + path: DavPath, +} + +impl WebdavFile { + pub fn new(op: Operator, path: DavPath) -> Self { + Self { op, path } + } } impl DavFile for WebdavFile { diff --git a/bin/oay/src/services/webdav/webdav_metadata.rs b/integrations/dav-server/src/webdav_metadata.rs similarity index 100% rename from bin/oay/src/services/webdav/webdav_metadata.rs rename to integrations/dav-server/src/webdav_metadata.rs diff --git a/integrations/dav-server-fs-opendal/src/webdavfs.rs b/integrations/dav-server/src/webdavfs.rs similarity index 97% rename from integrations/dav-server-fs-opendal/src/webdavfs.rs rename to integrations/dav-server/src/webdavfs.rs index 392926c307a6..7c36ed6b7bc7 100644 --- a/integrations/dav-server-fs-opendal/src/webdavfs.rs +++ b/integrations/dav-server/src/webdavfs.rs @@ -46,14 +46,10 @@ impl DavFileSystem for WebdavFs { fn open<'a>( &'a self, path: &'a dav_server::davpath::DavPath, - options: dav_server::fs::OpenOptions, + _options: dav_server::fs::OpenOptions, ) -> dav_server::fs::FsFuture> { async move { - let file = WebdavFile { - op: self.op.clone(), - path: path.clone(), - options, - }; + let file = WebdavFile::new(self.op.clone(), path.clone()); Ok(Box::new(file) as Box) } .boxed() diff --git a/integrations/dav-server-fs-opendal/tests/test.rs b/integrations/dav-server/tests/test.rs similarity index 89% rename from integrations/dav-server-fs-opendal/tests/test.rs rename to integrations/dav-server/tests/test.rs index ece739514e0e..3f8c9d7f1d20 100644 --- a/integrations/dav-server-fs-opendal/tests/test.rs +++ b/integrations/dav-server/tests/test.rs @@ -16,9 +16,9 @@ // under the License. use anyhow::Result; -use dav_server_fs_opendal::davpath::DavPath; -use dav_server_fs_opendal::fs::DavFileSystem; -use dav_server_fs_opendal::webdavfs::WebdavFs; +use dav_server::davpath::DavPath; +use dav_server::fs::DavFileSystem; +use dav_server_opendalfs::webdavfs::WebdavFs; use opendal::services::Fs; use opendal::Operator; From e993b7fcaf9eda9a2700c98756fff0b9fb6ccec8 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Tue, 19 Sep 2023 22:01:27 +0800 Subject: [PATCH 4/9] update --- Cargo.lock | 17 ++++ Cargo.toml | 1 + .../src/services/webdav/webdav_dir_entry.rs | 54 +++++++++++ bin/oay/src/services/webdav/webdav_file.rs | 94 +++++++++++++++++++ .../src/services/webdav/webdav_metadata.rs | 62 ++++++++++++ 5 files changed, 228 insertions(+) create mode 100644 bin/oay/src/services/webdav/webdav_dir_entry.rs create mode 100644 bin/oay/src/services/webdav/webdav_file.rs create mode 100644 bin/oay/src/services/webdav/webdav_metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 10e8741c7006..5543c22c9d69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1380,6 +1380,23 @@ dependencies = [ "xmltree", ] +[[package]] +name = "dav-server-opendalfs" +version = "0.39.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "dav-server", + "dirs 5.0.1", + "futures", + "futures-util", + "opendal", + "quick-xml", + "serde", + "tokio", +] + [[package]] name = "deadpool" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index b761b8b48d55..ac45ee858484 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "bin/oay", "integrations/object_store", + "integrations/dav-server", ] resolver = "2" diff --git a/bin/oay/src/services/webdav/webdav_dir_entry.rs b/bin/oay/src/services/webdav/webdav_dir_entry.rs new file mode 100644 index 000000000000..c3836db3ef48 --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_dir_entry.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavDirEntry; +use futures::FutureExt; +use opendal::Entry; +use opendal::Operator; + +use super::webdav_file::convert_error; +use super::webdav_metadata::WebdavMetaData; + +pub struct WebDAVDirEntry { + dir_entry: Entry, + op: Operator, +} + +impl DavDirEntry for WebDAVDirEntry { + fn name(&self) -> Vec { + self.dir_entry.name().as_bytes().to_vec() + } + + fn metadata(&self) -> dav_server::fs::FsFuture> { + async move { + self.op + .stat(self.dir_entry.path()) + .await + .map(|metadata| { + Box::new(WebdavMetaData::new(metadata)) as Box + }) + .map_err(convert_error) + } + .boxed() + } +} + +impl WebDAVDirEntry { + pub fn new(dir_entry: Entry, op: Operator) -> Self { + WebDAVDirEntry { dir_entry, op } + } +} diff --git a/bin/oay/src/services/webdav/webdav_file.rs b/bin/oay/src/services/webdav/webdav_file.rs new file mode 100644 index 000000000000..53e99c07fceb --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_file.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::io::SeekFrom; + +use bytes::Bytes; +use dav_server::davpath::DavPath; +use dav_server::fs::DavFile; +use dav_server::fs::DavMetaData; +use dav_server::fs::FsFuture; +use dav_server::fs::OpenOptions; +use futures::FutureExt; +use opendal::Operator; + +use super::webdav_metadata::WebdavMetaData; + +#[derive(Debug)] +pub struct WebdavFile { + pub op: Operator, + pub path: DavPath, + pub options: OpenOptions, +} + +impl DavFile for WebdavFile { + fn read_bytes(&mut self, count: usize) -> FsFuture { + async move { + let file_path = self.path.as_url_string(); + self.op + .read_with(&file_path) + .range(0..count as u64) + .await + .map(Bytes::from) + .map_err(convert_error) + } + .boxed() + } + + fn metadata(&mut self) -> FsFuture> { + async move { + self.op + .stat(self.path.as_url_string().as_str()) + .await + .map(|opendal_metadata| { + Box::new(WebdavMetaData::new(opendal_metadata)) as Box + }) + .map_err(convert_error) + } + .boxed() + } + + fn write_buf(&mut self, buf: Box) -> FsFuture<()> { + self.write_bytes(Bytes::copy_from_slice(buf.chunk())) + } + + fn write_bytes(&mut self, buf: Bytes) -> FsFuture<()> { + async move { + let file_path = self.path.as_url_string(); + self.op.write(&file_path, buf).await.map_err(convert_error) + } + .boxed() + } + + fn seek(&mut self, _pos: SeekFrom) -> FsFuture { + futures_util::future::err(dav_server::fs::FsError::NotImplemented).boxed() + } + + fn flush(&mut self) -> FsFuture<()> { + futures_util::future::ok(()).boxed() + } +} + +pub fn convert_error(opendal_error: opendal::Error) -> dav_server::fs::FsError { + match opendal_error.kind() { + opendal::ErrorKind::AlreadyExists | opendal::ErrorKind::IsSameFile => { + dav_server::fs::FsError::Exists + } + opendal::ErrorKind::NotFound => dav_server::fs::FsError::NotFound, + _ => dav_server::fs::FsError::GeneralFailure, + } +} diff --git a/bin/oay/src/services/webdav/webdav_metadata.rs b/bin/oay/src/services/webdav/webdav_metadata.rs new file mode 100644 index 000000000000..03295125d5ee --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_metadata.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavMetaData; +use dav_server::fs::FsError; +use opendal::Metadata; + +#[derive(Debug, Clone)] +pub struct WebdavMetaData { + metadata: Metadata, +} + +impl WebdavMetaData { + pub fn new(metadata: Metadata) -> Self { + WebdavMetaData { metadata } + } +} + +impl DavMetaData for WebdavMetaData { + fn len(&self) -> u64 { + self.metadata.content_length() + } + + fn modified(&self) -> dav_server::fs::FsResult { + match self.metadata.last_modified() { + Some(t) => Ok(t.into()), + None => Err(FsError::GeneralFailure), + } + } + + fn is_dir(&self) -> bool { + self.metadata.is_dir() + } + + fn is_file(&self) -> bool { + self.metadata.is_file() + } + + fn etag(&self) -> Option { + self.metadata.etag().map(|s| s.to_string()) + } + + fn status_changed(&self) -> dav_server::fs::FsResult { + self.metadata + .last_modified() + .map_or(Err(FsError::GeneralFailure), |t| Ok(t.into())) + } +} From 98b830a57c2eaeee8098510537b41c29eee40af6 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Tue, 19 Sep 2023 22:14:32 +0800 Subject: [PATCH 5/9] upgrade version --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5543c22c9d69..8e5bdde9d4a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1382,7 +1382,7 @@ dependencies = [ [[package]] name = "dav-server-opendalfs" -version = "0.39.0" +version = "0.40.0" dependencies = [ "anyhow", "bytes", From 909d819fc89ffb181874d9078930b7d8f19e6d96 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Thu, 21 Sep 2023 16:19:08 +0800 Subject: [PATCH 6/9] update --- Cargo.lock | 2 +- .../src/{webdav_dir_entry.rs => dir_entry.rs} | 4 ++-- .../dav-server/src/{webdav_file.rs => file.rs} | 2 +- integrations/dav-server/src/lib.rs | 10 +++++----- .../src/{webdav_metadata.rs => metadata.rs} | 0 .../src/{webdavfs.rs => opendalfs.rs} | 18 +++++++++--------- integrations/dav-server/tests/test.rs | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) rename integrations/dav-server/src/{webdav_dir_entry.rs => dir_entry.rs} (95%) rename integrations/dav-server/src/{webdav_file.rs => file.rs} (98%) rename integrations/dav-server/src/{webdav_metadata.rs => metadata.rs} (100%) rename integrations/dav-server/src/{webdavfs.rs => opendalfs.rs} (95%) diff --git a/Cargo.lock b/Cargo.lock index 8e5bdde9d4a9..d6dc0f2702a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3389,7 +3389,7 @@ dependencies = [ "bytes", "chrono", "clap 4.3.10", - "dav-server", + "dav-server-opendalfs", "dirs 5.0.1", "futures", "futures-util", diff --git a/integrations/dav-server/src/webdav_dir_entry.rs b/integrations/dav-server/src/dir_entry.rs similarity index 95% rename from integrations/dav-server/src/webdav_dir_entry.rs rename to integrations/dav-server/src/dir_entry.rs index c3836db3ef48..955266546cf4 100644 --- a/integrations/dav-server/src/webdav_dir_entry.rs +++ b/integrations/dav-server/src/dir_entry.rs @@ -20,8 +20,8 @@ use futures::FutureExt; use opendal::Entry; use opendal::Operator; -use super::webdav_file::convert_error; -use super::webdav_metadata::WebdavMetaData; +use super::file::convert_error; +use super::metadata::WebdavMetaData; pub struct WebDAVDirEntry { dir_entry: Entry, diff --git a/integrations/dav-server/src/webdav_file.rs b/integrations/dav-server/src/file.rs similarity index 98% rename from integrations/dav-server/src/webdav_file.rs rename to integrations/dav-server/src/file.rs index 3406b1dc6d9c..c8983d1fc258 100644 --- a/integrations/dav-server/src/webdav_file.rs +++ b/integrations/dav-server/src/file.rs @@ -25,7 +25,7 @@ use dav_server::fs::FsFuture; use futures::FutureExt; use opendal::Operator; -use super::webdav_metadata::WebdavMetaData; +use super::metadata::WebdavMetaData; #[derive(Debug)] pub struct WebdavFile { diff --git a/integrations/dav-server/src/lib.rs b/integrations/dav-server/src/lib.rs index ca642b2fbeba..6436fd7370b4 100644 --- a/integrations/dav-server/src/lib.rs +++ b/integrations/dav-server/src/lib.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -mod webdav_dir_entry; -mod webdav_file; -mod webdav_metadata; -pub mod webdavfs; +mod dir_entry; +mod file; +mod metadata; +mod opendalfs; -pub use dav_server::DavHandler; +pub use opendalfs::OpendalFs; diff --git a/integrations/dav-server/src/webdav_metadata.rs b/integrations/dav-server/src/metadata.rs similarity index 100% rename from integrations/dav-server/src/webdav_metadata.rs rename to integrations/dav-server/src/metadata.rs diff --git a/integrations/dav-server/src/webdavfs.rs b/integrations/dav-server/src/opendalfs.rs similarity index 95% rename from integrations/dav-server/src/webdavfs.rs rename to integrations/dav-server/src/opendalfs.rs index 7c36ed6b7bc7..3e72fd89e8ea 100644 --- a/integrations/dav-server/src/webdavfs.rs +++ b/integrations/dav-server/src/opendalfs.rs @@ -32,17 +32,17 @@ use futures_util::StreamExt; use opendal::Lister; use opendal::Operator; -use super::webdav_file::convert_error; -use super::webdav_file::WebdavFile; -use super::webdav_metadata::WebdavMetaData; -use crate::webdav_dir_entry::WebDAVDirEntry; +use super::file::convert_error; +use super::file::WebdavFile; +use super::metadata::WebdavMetaData; +use crate::dir_entry::WebDAVDirEntry; #[derive(Clone)] -pub struct WebdavFs { +pub struct OpendalFs { pub op: Operator, } -impl DavFileSystem for WebdavFs { +impl DavFileSystem for OpendalFs { fn open<'a>( &'a self, path: &'a dav_server::davpath::DavPath, @@ -175,9 +175,9 @@ impl DavFileSystem for WebdavFs { } } -impl WebdavFs { - pub fn new(op: Operator) -> Box { - Box::new(WebdavFs { op }) +impl OpendalFs { + pub fn new(op: Operator) -> Box { + Box::new(OpendalFs { op }) } } diff --git a/integrations/dav-server/tests/test.rs b/integrations/dav-server/tests/test.rs index 3f8c9d7f1d20..0ad79f720762 100644 --- a/integrations/dav-server/tests/test.rs +++ b/integrations/dav-server/tests/test.rs @@ -18,7 +18,7 @@ use anyhow::Result; use dav_server::davpath::DavPath; use dav_server::fs::DavFileSystem; -use dav_server_opendalfs::webdavfs::WebdavFs; +use dav_server_opendalfs::OpendalFs; use opendal::services::Fs; use opendal::Operator; @@ -29,7 +29,7 @@ async fn test() -> Result<()> { let op = Operator::new(builder)?.finish(); - let webdavfs = WebdavFs::new(op); + let webdavfs = OpendalFs::new(op); let metadata = webdavfs .metadata(&DavPath::new("/").unwrap()) From 836ad64d1c169764b4302d4c04ea1b34f70bc9a2 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Thu, 21 Sep 2023 16:54:29 +0800 Subject: [PATCH 7/9] update Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d6dc0f2702a5..8e5bdde9d4a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3389,7 +3389,7 @@ dependencies = [ "bytes", "chrono", "clap 4.3.10", - "dav-server-opendalfs", + "dav-server", "dirs 5.0.1", "futures", "futures-util", From 8119c570524e06724278017493e4ec65f7bbc52c Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Fri, 22 Sep 2023 21:42:38 +0800 Subject: [PATCH 8/9] clean code --- integrations/dav-server/src/opendalfs.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/integrations/dav-server/src/opendalfs.rs b/integrations/dav-server/src/opendalfs.rs index 3e72fd89e8ea..92a2d7783749 100644 --- a/integrations/dav-server/src/opendalfs.rs +++ b/integrations/dav-server/src/opendalfs.rs @@ -16,8 +16,7 @@ // under the License. use std::path::Path; -use std::pin::Pin; -use std::task::Poll::Pending; +use std::task::ready; use std::task::Poll::Ready; use dav_server::davpath::DavPath; @@ -194,17 +193,12 @@ impl Stream for DavStream { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { let dav_stream = self.get_mut(); - let lister = Pin::new(&mut dav_stream.lister).get_mut(); - - match Pin::new(lister).poll_next(cx) { - Ready(entry) => match entry { - Some(entry) => { - let webdav_entry = WebDAVDirEntry::new(entry.unwrap(), dav_stream.op.clone()); - Ready(Some(Box::new(webdav_entry) as Box)) - } - None => Ready(None), - }, - Pending => Pending, + match ready!(dav_stream.lister.poll_next_unpin(cx)) { + Some(entry) => { + let webdav_entry = WebDAVDirEntry::new(entry.unwrap(), dav_stream.op.clone()); + Ready(Some(Box::new(webdav_entry) as Box)) + } + None => Ready(None), } } } From b1c4356257f4ac73a512199c5bab830653314523 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Sat, 7 Oct 2023 18:43:07 +0800 Subject: [PATCH 9/9] update Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8a2a2803769d..ef379ea43123 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1568,7 +1568,7 @@ dependencies = [ "futures", "futures-util", "opendal", - "quick-xml", + "quick-xml 0.29.0", "serde", "tokio", ]