From 05d6b84418bb51c794b5d8aaf9763666c542c5ed Mon Sep 17 00:00:00 2001 From: Zhengke Zhou <36791902+zzzk1@users.noreply.github.com> Date: Thu, 27 Jun 2024 22:50:03 +0800 Subject: [PATCH] refactor(core): Remove unused layer `MadsimLayer` (#4788) --- core/Cargo.lock | 121 +---------- core/Cargo.toml | 4 - core/src/layers/madsim.rs | 437 -------------------------------------- core/src/layers/mod.rs | 7 - 4 files changed, 2 insertions(+), 567 deletions(-) delete mode 100644 core/src/layers/madsim.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 7a365c345c96..3f81fdbb2e47 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2400,12 +2400,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "downcast-rs" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" - [[package]] name = "dtoa" version = "1.0.9" @@ -4145,49 +4139,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "madsim" -version = "0.2.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88753ddf8d3cd43b9cf71a93626dd9aad3c24086a04420beb31922e1f856d02" -dependencies = [ - "ahash 0.8.11", - "async-channel 2.3.1", - "async-stream", - "async-task", - "bincode", - "bytes", - "downcast-rs", - "futures-util", - "lazy_static", - "libc", - "madsim-macros", - "naive-timer", - "panic-message", - "rand 0.8.5", - "rand_xoshiro", - "rustversion", - "serde", - "spin 0.9.8", - "tokio", - "tokio-util", - "toml", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "madsim-macros" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "match_cfg" version = "0.1.0" @@ -4549,12 +4500,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "naive-timer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" - [[package]] name = "nanoid" version = "0.4.0" @@ -4798,7 +4743,6 @@ dependencies = [ "http 1.1.0", "libtest-mimic", "log", - "madsim", "md-5", "metrics", "mini-moka", @@ -5108,12 +5052,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "panic-message" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" - [[package]] name = "parking" version = "2.2.0" @@ -5970,15 +5908,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "raw-cpuid" version = "11.0.2" @@ -6906,15 +6835,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -7954,26 +7874,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.14", -] - [[package]] name = "toml_datetime" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" -dependencies = [ - "serde", -] [[package]] name = "toml_edit" @@ -7983,7 +7888,7 @@ checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.2.6", "toml_datetime", - "winnow 0.5.40", + "winnow", ] [[package]] @@ -7994,20 +7899,7 @@ checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ "indexmap 2.2.6", "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" -dependencies = [ - "indexmap 2.2.6", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.6.13", + "winnow", ] [[package]] @@ -8997,15 +8889,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index 26678b7aea47..abdb7bae277f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -83,8 +83,6 @@ layers-metrics = ["dep:metrics"] layers-prometheus = ["dep:prometheus"] # Enable layers prometheus support, with prometheus-client crate layers-prometheus-client = ["dep:prometheus-client"] -# Enable layers madsim support -layers-madsim = ["dep:madsim"] # Enable layers minitrace support. layers-minitrace = ["dep:minitrace"] # Enable layers tracing support. @@ -355,8 +353,6 @@ async-backtrace = { version = "0.2.6", optional = true } await-tree = { version = "0.1.1", optional = true } # for layers-throttle governor = { version = "0.6.0", optional = true, features = ["std"] } -# for layers-madsim -madsim = { version = "0.2.21", optional = true } # for layers-metrics metrics = { version = "0.20", optional = true } # for layers-minitrace diff --git a/core/src/layers/madsim.rs b/core/src/layers/madsim.rs deleted file mode 100644 index 5244820effc0..000000000000 --- a/core/src/layers/madsim.rs +++ /dev/null @@ -1,437 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This module requires to be work under `cfg(madsim)` so we will allow -// dead code and unused variables. -#![allow(unused_imports)] -#![allow(unused_variables)] -#![allow(dead_code)] - -use std::any::Any; -use std::cmp::min; -use std::collections::HashMap; -use std::fmt::Debug; -use std::future::Future; -use std::io::Result; -use std::io::SeekFrom; -use std::net::SocketAddr; -use std::sync::Arc; -use std::sync::Mutex; -use std::task::Context; -use std::task::Poll; - -use bytes::Bytes; -#[cfg(madsim)] -use madsim::net::Endpoint; -#[cfg(madsim)] -use madsim::net::Payload; - -use crate::raw::*; -use crate::*; - -/// Add deterministic simulation for async operations, powered by [`madsim`](https://docs.rs/madsim/latest/madsim/). -/// -/// # Note -/// -/// - blocking operations are not supported, as [`madsim`](https://docs.rs/madsim/latest/madsim/) is async only. -/// -/// -/// # Examples -/// -/// ```no_build -/// use std::time::Duration; -/// -/// use madsim::net::NetSim; -/// use madsim::runtime::Handle; -/// use madsim::time::sleep; -/// use opendal::layers::MadsimLayer; -/// use opendal::layers::MadsimServer; -/// use opendal::services; -/// use opendal::Operator; -/// -/// #[cfg(madsim)] -/// #[madsim::test] -/// async fn deterministic_simulation_test() { -/// let handle = Handle::current(); -/// let ip1 = "10.0.0.1".parse().unwrap(); -/// let ip2 = "10.0.0.2".parse().unwrap(); -/// let server_addr = "10.0.0.1:2379".parse().unwrap(); -/// let server = handle.create_node().name("server").ip(ip1).build(); -/// let client = handle.create_node().name("client").ip(ip2).build(); -/// -/// server.spawn(async move { -/// SimServer::serve(server_addr).await.unwrap(); -/// }); -/// sleep(Duration::from_secs(1)).await; -/// -/// let handle = client.spawn(async move { -/// let mut builder = services::Fs::default(); -/// builder.root("."); -/// let op = Operator::new(builder) -/// .unwrap() -/// .layer(MadsimLayer::new(server_addr)) -/// .finish(); -/// -/// let path = "hello.txt"; -/// let data = "Hello, World!"; -/// op.write(path, data).await.unwrap(); -/// assert_eq!(data.as_bytes(), op.read(path).await.unwrap()); -/// }); -/// handle.await.unwrap(); -/// } -/// ``` -/// To enable logging output, please set `RUSTFLAGS="--cfg madsim"`: -/// ```shell -/// RUSTFLAGS="--cfg madsim" cargo test -/// ``` -#[derive(Debug, Copy, Clone)] -pub struct MadsimLayer { - #[cfg(madsim)] - addr: SocketAddr, -} - -impl MadsimLayer { - /// Create new madsim layer - pub fn new(endpoint: &str) -> Self { - #[cfg(madsim)] - { - Self { - addr: endpoint.parse().unwrap(), - } - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } -} - -impl Layer for MadsimLayer { - type LayeredAccess = MadsimAccessor; - - fn layer(&self, _: A) -> Self::LayeredAccess { - #[cfg(madsim)] - { - MadsimAccessor { addr: self.addr } - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } -} - -#[derive(Debug)] -pub struct MadsimAccessor { - #[cfg(madsim)] - addr: SocketAddr, -} - -impl LayeredAccess for MadsimAccessor { - type Inner = (); - type Reader = MadsimReader; - type BlockingReader = (); - type Writer = MadsimWriter; - type BlockingWriter = (); - type Lister = MadsimLister; - type BlockingLister = (); - - fn inner(&self) -> &Self::Inner { - &() - } - - fn metadata(&self) -> AccessorInfo { - let mut info = AccessorInfo::default(); - info.set_name("madsim"); - - info.set_native_capability(Capability { - read: true, - write: true, - ..Default::default() - }); - - info - } - - async fn read(&self, path: &str, args: OpRead) -> crate::Result<(RpRead, Self::Reader)> { - #[cfg(madsim)] - { - let req = Request::Read(path.to_string(), args); - let ep = Endpoint::connect(self.addr) - .await - .expect("fail to connect to sim server"); - let (tx, mut rx) = ep - .connect1(self.addr) - .await - .expect("fail to connect1 to sim server"); - tx.send(Box::new(req)) - .await - .expect("fail to send request to sim server"); - let resp = rx - .recv() - .await - .expect("fail to recv response from sim server"); - let resp = resp - .downcast::() - .expect("fail to downcast response to ReadResponse"); - let content_length = resp.data.as_ref().map(|b| b.len()).unwrap_or(0); - Ok((RpRead::new(), MadsimReader { data: resp.data })) - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } - - async fn write(&self, path: &str, args: OpWrite) -> crate::Result<(RpWrite, Self::Writer)> { - #[cfg(madsim)] - { - Ok(( - RpWrite::default(), - MadsimWriter { - path: path.to_string(), - args, - addr: self.addr, - }, - )) - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } - - async fn list(&self, path: &str, args: OpList) -> crate::Result<(RpList, Self::Lister)> { - Err(Error::new( - ErrorKind::Unsupported, - "will be supported in the future", - )) - } - - fn blocking_read( - &self, - path: &str, - args: OpRead, - ) -> crate::Result<(RpRead, Self::BlockingReader)> { - Err(Error::new( - ErrorKind::Unsupported, - "will not be supported in MadsimLayer", - )) - } - - fn blocking_write( - &self, - path: &str, - args: OpWrite, - ) -> crate::Result<(RpWrite, Self::BlockingWriter)> { - Err(Error::new( - ErrorKind::Unsupported, - "will not be supported in MadsimLayer", - )) - } - - fn blocking_list( - &self, - path: &str, - args: OpList, - ) -> crate::Result<(RpList, Self::BlockingLister)> { - Err(Error::new( - ErrorKind::Unsupported, - "will not be supported in MadsimLayer", - )) - } -} - -pub struct MadsimReader { - data: Option, -} - -impl oio::Read for MadsimReader { - async fn read(&mut self) -> crate::Result { - if let Some(data) = self.data.take() { - Ok(Buffer::from(data)) - } else { - Ok(Buffer::new()) - } - } -} - -pub struct MadsimWriter { - #[cfg(madsim)] - path: String, - #[cfg(madsim)] - args: OpWrite, - #[cfg(madsim)] - addr: SocketAddr, -} - -impl oio::Write for MadsimWriter { - async fn write(&mut self, bs: Buffer) -> crate::Result { - #[cfg(madsim)] - { - let req = Request::Write(self.path.to_string(), bs); - let ep = Endpoint::bind(self.addr).await?; - let (tx, mut rx) = ep.connect1(self.addr).await?; - tx.send(Box::new(req)).await?; - rx.recv().await?; - Ok(()) - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } - - async fn abort(&mut self) -> crate::Result<()> { - Err(Error::new( - ErrorKind::Unsupported, - "will be supported in the future", - )) - } - - async fn close(&mut self) -> crate::Result<()> { - Ok(()) - } -} - -pub struct MadsimLister {} - -impl oio::List for MadsimLister { - async fn next(&mut self) -> crate::Result> { - Err(Error::new( - ErrorKind::Unsupported, - "will be supported in the future", - )) - } -} - -/// A simulated server.This an experimental feature, docs are not ready yet. -#[derive(Default, Clone)] -pub struct MadsimServer; - -impl MadsimServer { - /// Start serving as madsim server. - pub async fn serve(addr: SocketAddr) -> Result<()> { - #[cfg(madsim)] - { - let ep = Endpoint::bind(addr).await?; - let service = Arc::new(SimService::default()); - loop { - let (tx, mut rx, _) = ep.accept1().await?; - let service = service.clone(); - madsim::task::spawn(async move { - let request = *rx - .recv() - .await? - .downcast::() - .expect("invalid request"); - let response = match request { - Request::Read(path, args) => { - Box::new(service.read(&path, args).await) as Payload - } - Request::Write(path, args) => { - Box::new(service.write(&path, args).await) as Payload - } - }; - tx.send(response).await?; - Ok(()) as Result<()> - }); - } - } - #[cfg(not(madsim))] - { - unreachable!("madsim is not enabled") - } - } -} - -enum Request { - Read(String, OpRead), - Write(String, Bytes), -} - -#[derive(Default)] -pub struct SimService { - inner: Mutex>, -} - -impl SimService { - async fn read(&self, path: &str, args: OpRead) -> ReadResponse { - let inner = self.inner.lock().unwrap(); - let data = inner.get(path); - ReadResponse { - data: data.cloned(), - } - } - - async fn write(&self, path: &str, data: Bytes) -> WriteResponse { - let mut inner = self.inner.lock().unwrap(); - inner.insert(path.to_string(), data); - WriteResponse {} - } -} - -struct ReadResponse { - data: Option, -} - -struct WriteResponse {} - -#[cfg(test)] -#[cfg(madsim)] -mod test { - use std::time::Duration; - - use madsim::runtime::Handle; - use madsim::time::sleep; - - use super::*; - use crate::services; - use crate::Operator; - - #[madsim::test] - async fn test_madsim_layer() { - let handle = Handle::current(); - let ip1 = "10.0.0.1".parse().unwrap(); - let ip2 = "10.0.0.2".parse().unwrap(); - let server_addr = "10.0.0.1:2379"; - let server = handle.create_node().name("server").ip(ip1).build(); - let client = handle.create_node().name("client").ip(ip2).build(); - - server.spawn(async move { - MadsimServer::serve(server_addr.parse().unwrap()) - .await - .unwrap(); - }); - sleep(Duration::from_secs(1)).await; - - let handle = client.spawn(async move { - let mut builder = services::Fs::default(); - builder.root("."); - let op = Operator::new(builder) - .unwrap() - .layer(MadsimLayer::new(server_addr)) - .finish(); - - let path = "hello.txt"; - let data = "Hello, World!"; - op.write(path, data).await.unwrap(); - assert_eq!(data.as_bytes(), op.read(path).await.unwrap()); - }); - handle.await.unwrap(); - } -} diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs index 4f5d65a57539..9adcde1ae924 100644 --- a/core/src/layers/mod.rs +++ b/core/src/layers/mod.rs @@ -77,13 +77,6 @@ mod minitrace; #[cfg(feature = "layers-minitrace")] pub use self::minitrace::MinitraceLayer; -#[cfg(feature = "layers-madsim")] -mod madsim; -#[cfg(feature = "layers-madsim")] -pub use self::madsim::MadsimLayer; -#[cfg(feature = "layers-madsim")] -pub use self::madsim::MadsimServer; - #[cfg(feature = "layers-otel-trace")] mod oteltrace; #[cfg(feature = "layers-otel-trace")]