From ed90d02ca363564692f72783e76c578c4ec3548f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 8 Oct 2022 23:21:33 +0800 Subject: [PATCH] Bump to version 0.19.0 (#821) * Bump to version 0.19.0 Signed-off-by: Xuanwo * Add upgrade Signed-off-by: Xuanwo * Fix oss Signed-off-by: Xuanwo * Fix build Signed-off-by: Xuanwo * Fix docs Signed-off-by: Xuanwo * Add docs Signed-off-by: Xuanwo Signed-off-by: Xuanwo --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ Cargo.toml | 12 +++++------- README.md | 3 ++- docs/services/oss.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/upgrade.md | 10 ++++++++++ oay/Cargo.lock | 8 ++++---- oay/Cargo.toml | 4 ++-- oli/Cargo.lock | 8 ++++---- oli/Cargo.toml | 4 ++-- src/layers/mod.rs | 2 -- src/lib.rs | 2 ++ src/object/entry.rs | 2 +- src/object/iterate.rs | 2 +- src/object/stream.rs | 2 +- 14 files changed, 105 insertions(+), 25 deletions(-) create mode 100644 docs/services/oss.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bd12cee843d2..012f1b0dd8ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [v0.19.0] - 2022-10-08 + +### Added + +- feat: Implement object page stream for services like s3 (#787) +- RFC-0793: Generic KV Services (#793) +- feat(services/kv): Implement Scoped Key (#796) +- feat: Add scan in KeyValueAccessor (#797) +- feat: Implement basic kv services support (#799) +- feat: Introduce kv adapter for opendal (#802) +- feat: Add integration test for redis (#804) +- feat: Add OSS Service Support (#801) +- feat: Add integration tests for OSS (#814) + +### Changed + +- refactor: Move object to mod (#786) +- refactor: Implement azblob dir stream based on ObjectPageStream (#790) +- refactor: Implement memory services by generic kv (#800) +- refactor: Don't expose backend to users (#816) +- tests: allow running tests when env is `true` (#818) +- refactor: Remove deprecated type aliases (#819) +- test: list rich dir (#820) + +### Fixed + +- fix(services/redis): MATCH can't handle correctly (#803) +- fix: Disable ipfs redirection (#809) +- fix(services/ipfs): Use ipfs files API to copy data (#811) +- fix(services/hdfs): Allow retrying would block (#815) + ## [v0.18.2] - 2022-10-01 ### Added @@ -810,6 +841,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). Hello, OpenDAL! +[v0.19.0]: https://github.com/datafuselabs/opendal/compare/v0.18.2...v0.19.0 [v0.18.2]: https://github.com/datafuselabs/opendal/compare/v0.18.1...v0.18.2 [v0.18.1]: https://github.com/datafuselabs/opendal/compare/v0.18.0...v0.18.1 [v0.18.0]: https://github.com/datafuselabs/opendal/compare/v0.17.4...v0.18.0 diff --git a/Cargo.toml b/Cargo.toml index a83796b8ef96..4b747856d17f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ keywords = ["storage", "fs", "s3", "azblob", "gcs"] license = "Apache-2.0" name = "opendal" repository = "https://github.com/datafuselabs/opendal" -version = "0.18.2" +version = "0.19.0" [package.metadata.docs.rs] all-features = true @@ -22,10 +22,7 @@ compress = ["async-compression"] serde = [] # Enable all layers. -layers-all = [ - "layers-metrics", - "layers-tracing", -] +layers-all = ["layers-metrics", "layers-tracing"] # Enable layers metrics support layers-metrics = ["metrics"] # Enable layers retry support. @@ -117,7 +114,8 @@ redis = { version = "0.21", features = [ "tokio-comp", "connection-manager", ], optional = true } -reqsign = "0.5" +# Make sure reqsign version is larger than 0.5.2 +reqsign = "0.5.2" reqwest = { version = "0.11", features = [ "multipart", "stream", @@ -125,7 +123,7 @@ reqwest = { version = "0.11", features = [ ], default-features = false } serde = { version = "1", features = ["derive"] } serde_json = "1" -suppaftp = { version = "4", features = ["async-secure"], optional = true } +suppaftp = { version = "=4.4", features = ["async-secure"], optional = true } thiserror = "1" time = { version = "0.3", features = ["serde"] } tokio = { version = "1.20", features = ["fs"] } diff --git a/README.md b/README.md index 54c581b5f65c..409c446efd0d 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ You may be looking for: - [ipmfs](https://opendal.databend.rs/opendal/services/ipmfs/index.html): [InterPlanetary File System](https://ipfs.tech/) MFS API support. - [memory](https://opendal.databend.rs/opendal/services/memory/index.html): In memory backend. - [obs](https://opendal.databend.rs/opendal/services/obs/index.html): [Huawei Cloud Object Storage](https://www.huaweicloud.com/intl/en-us/product/obs.html) Service (OBS). -- [redis](https://opendal.databend.rs/opendal/services/redis/index.html): [Redis](https://redis.io/) services support. +- [oss](https://opendal.databend.rs/opendal/services/oss/index.html): [Aliyun Object Storage Service](https://www.aliyun.com/product/oss) (OSS). +- [redis](https://opendal.databend.rs/opendal/services/redis/index.html): [Redis](https://redis.io/) services support. - [s3](https://opendal.databend.rs/opendal/services/s3/index.html): [AWS S3](https://aws.amazon.com/s3/) alike services. ## Features diff --git a/docs/services/oss.md b/docs/services/oss.md new file mode 100644 index 000000000000..b226c5e00847 --- /dev/null +++ b/docs/services/oss.md @@ -0,0 +1,39 @@ +# Use OSS as backend + +This page provides some examples for using Aliyun OSS as backend. + +We can run this example via: + +```shell +cargo run --example oss +``` + +## Example + +### Via Environment Variables + +Available environment variables: + +- `OPENDAL_OSS_ROOT`: root path, default: / +- `OPENDAL_OSS_BUCKET`: bukcet name, required. +- `OPENDAL_OSS_ENDPOINT`: endpoint of oss service +- `OPENDAL_OSS_ACCESS_KEY_ID`: access key id of oss service, could be auto detected. +- `OPENDAL_OSS_ACCESS_KEY_SECRET`: secret access key of oss service, could be auto detected. + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; + +#[tokio::main] +async fn main() -> Result<()> { + // init operator from env vars + let _op = Operator::from_env(Scheme::Oss)?; +} +``` + +### Via Builder + +```rust +{{#include ../../examples/oss.rs:15:}} +``` diff --git a/docs/upgrade.md b/docs/upgrade.md index 93fa35a2f93d..65cfac25dd72 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -2,6 +2,16 @@ This document intends to record upgrade and migrate procedures while OpenDAL meets breaking changes. +## Upgrade to v0.19 + +OpenDAL deprecate some features: + +- `serde`: We will enable it by default. +- `layers-retry`: We will enable retry support by default. +- `layers-metadata-cache`: We will enable it by default. + +Deprecated types like `DirEntry` has been removed. + ## Upgrade to v0.18 OpenDAL v0.18 introduces the following breaking changes: diff --git a/oay/Cargo.lock b/oay/Cargo.lock index c18b1154c6e4..8894639c5598 100644 --- a/oay/Cargo.lock +++ b/oay/Cargo.lock @@ -1061,7 +1061,7 @@ dependencies = [ [[package]] name = "oay" -version = "0.18.1" +version = "0.19.0" dependencies = [ "actix-web", "anyhow", @@ -1083,7 +1083,7 @@ checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "opendal" -version = "0.18.2" +version = "0.19.0" dependencies = [ "anyhow", "async-compat", @@ -1313,9 +1313,9 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "reqsign" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c338306df4ba0010b78f1c4469bf034313ad46b2c43020ec448432e5e07fab26" +checksum = "a082aef1e547f31ddde9db1eb92e983ed92dbd521678709ba2574cf768c919ef" dependencies = [ "anyhow", "backon", diff --git a/oay/Cargo.toml b/oay/Cargo.toml index c0aba70681ab..119494512c14 100644 --- a/oay/Cargo.toml +++ b/oay/Cargo.toml @@ -8,7 +8,7 @@ keywords = ["storage", "data", "s3", "fs", "azblob"] license = "Apache-2.0" name = "oay" repository = "https://github.com/datafuselabs/opendal" -version = "0.18.1" +version = "0.19.0" [dependencies] actix-web = "4" @@ -17,7 +17,7 @@ clap = { version = "4", features = ["cargo"] } env_logger = "0.9" futures = "0.3" log = "0.4" -opendal = "0.18" +opendal = "0.19" percent-encoding = "2" sluice = "0.5" tokio = { version = "1.20", features = ["rt-multi-thread", "macros"] } diff --git a/oli/Cargo.lock b/oli/Cargo.lock index 9fa3cc0b409a..5531b658fa7b 100644 --- a/oli/Cargo.lock +++ b/oli/Cargo.lock @@ -806,7 +806,7 @@ dependencies = [ [[package]] name = "oli" -version = "0.18.1" +version = "0.19.0" dependencies = [ "anyhow", "assert_cmd", @@ -826,7 +826,7 @@ checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "opendal" -version = "0.18.2" +version = "0.19.0" dependencies = [ "anyhow", "async-compat", @@ -1086,9 +1086,9 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "reqsign" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c338306df4ba0010b78f1c4469bf034313ad46b2c43020ec448432e5e07fab26" +checksum = "a082aef1e547f31ddde9db1eb92e983ed92dbd521678709ba2574cf768c919ef" dependencies = [ "anyhow", "backon", diff --git a/oli/Cargo.toml b/oli/Cargo.toml index 407289310178..a0e1fdca63d2 100644 --- a/oli/Cargo.toml +++ b/oli/Cargo.toml @@ -8,14 +8,14 @@ keywords = ["storage", "data", "s3", "fs", "azblob"] license = "Apache-2.0" name = "oli" repository = "https://github.com/datafuselabs/opendal" -version = "0.18.1" +version = "0.19.0" [dependencies] anyhow = "1" clap = { version = "4", features = ["cargo", "string"] } env_logger = "0.9" log = "0.4" -opendal = "0.18" +opendal = "0.19" tokio = { version = "1.20", features = ["fs", "macros", "rt-multi-thread"] } [dev-dependencies] diff --git a/src/layers/mod.rs b/src/layers/mod.rs index 83e2ab45c19a..7fc63a1ac73d 100644 --- a/src/layers/mod.rs +++ b/src/layers/mod.rs @@ -33,9 +33,7 @@ pub use immutable_index::ImmutableIndexLayer; mod logging; pub use logging::LoggingLayer; -#[cfg(feature = "layers-metadata-cache")] mod metadata_cache; -#[cfg(feature = "layers-metadata-cache")] pub use metadata_cache::MetadataCacheLayer; #[cfg(feature = "layers-metrics")] diff --git a/src/lib.rs b/src/lib.rs index b38cca4c80be..18d2db092c11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,8 @@ //! | [ipmfs][crate::services::ipmfs] | IPFS Mutable File System support. | //! | [memory][crate::services::memory] | In memory backend support. | //! | [obs][crate::services::obs] | Huawei Cloud OBS service. | +//! | [oss][crate::services::oss] | Aliyun Object Storage Service (OSS).| +//! | [redis][crate::services::redis] | Redis service. | //! | [s3][crate::services::s3] | AWS S3 alike services. | //! //! # Optional features diff --git a/src/object/entry.rs b/src/object/entry.rs index 720cf103fe92..640ab6dece02 100644 --- a/src/object/entry.rs +++ b/src/object/entry.rs @@ -26,7 +26,7 @@ use crate::Object; use crate::ObjectMetadata; use crate::ObjectMode; -/// ObjectEntry is returned by [`ObjectStream`][crate::ObjectStream] or [`ObjectIterate`][crate::ObjectIterate] during object list. +/// ObjectEntry is returned by `ObjectStream` or `ObjectIterate` during object list. /// /// Users can check returning object entry's mode or convert into an object without overhead. #[derive(Debug)] diff --git a/src/object/iterate.rs b/src/object/iterate.rs index dc48f5eaa351..f680535a551d 100644 --- a/src/object/iterate.rs +++ b/src/object/iterate.rs @@ -20,7 +20,7 @@ use crate::ObjectEntry; pub trait ObjectIterate: Iterator> {} impl ObjectIterate for T where T: Iterator> {} -/// ObjectIterator is a boxed dyn [`ObjectIterate`] +/// ObjectIterator is a boxed dyn `ObjectIterate` pub type ObjectIterator = Box; /// EmptyObjectIterator that always return None. diff --git a/src/object/stream.rs b/src/object/stream.rs index 5ee2aceaf9a6..8ee5aa1bee5e 100644 --- a/src/object/stream.rs +++ b/src/object/stream.rs @@ -33,7 +33,7 @@ use crate::ObjectEntry; pub trait ObjectStream: Stream> + Unpin + Send {} impl ObjectStream for T where T: Stream> + Unpin + Send {} -/// ObjectStreamer is a boxed dyn [`ObjectStream`] +/// ObjectStreamer is a boxed dyn `ObjectStream` pub type ObjectStreamer = Box; /// EmptyObjectStreamer that always return None.