From d479ac39efa9a88e10448ad81e47292b3b4ed688 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Wed, 25 Sep 2024 17:06:12 +0800 Subject: [PATCH] test(core): add tests for versioning (#5132) * test(core): add tests for versioning * add tests for version doesn't match * update comments * add stat_with_versioning, read_with_versioning, delete_with_versioning, list_with_versioning_capabilities * read/stat non-existent version returns NotFound, delete non-existent version returns Ok * fix docs --- core/src/services/s3/backend.rs | 5 +- core/src/types/capability.rs | 14 ++-- core/src/types/operator/operator_futures.rs | 2 +- core/tests/behavior/async_delete.rs | 71 ++++++++++++++++++- core/tests/behavior/async_read.rs | 67 +++++++++++++++++- core/tests/behavior/async_stat.rs | 78 +++++++++++++++++++-- 6 files changed, 224 insertions(+), 13 deletions(-) diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 745b292f2b5a..8073cbfbb929 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -902,14 +902,15 @@ impl Access for S3Backend { stat_with_override_cache_control: !self.core.disable_stat_with_override, stat_with_override_content_disposition: !self.core.disable_stat_with_override, stat_with_override_content_type: !self.core.disable_stat_with_override, + stat_with_version: self.core.enable_versioning, read: true, - read_with_if_match: true, read_with_if_none_match: true, read_with_override_cache_control: true, read_with_override_content_disposition: true, read_with_override_content_type: true, + read_with_version: self.core.enable_versioning, write: true, write_can_empty: true, @@ -932,6 +933,8 @@ impl Access for S3Backend { }, delete: true, + delete_with_version: self.core.enable_versioning, + copy: true, list: true, diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index cae69f04ab68..4a874b238771 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -61,12 +61,14 @@ pub struct Capability { pub stat_with_if_match: bool, /// If operator supports stat with if none match. pub stat_with_if_none_match: bool, - /// if operator supports read with override cache control. + /// if operator supports stat with override cache control. pub stat_with_override_cache_control: bool, - /// if operator supports read with override content disposition. + /// if operator supports stat with override content disposition. pub stat_with_override_content_disposition: bool, - /// if operator supports read with override content type. + /// if operator supports stat with override content type. pub stat_with_override_content_type: bool, + /// if operator supports stat with version. + pub stat_with_version: bool, /// If operator supports read. pub read: bool, @@ -80,6 +82,8 @@ pub struct Capability { pub read_with_override_content_disposition: bool, /// if operator supports read with override content type. pub read_with_override_content_type: bool, + /// if operator supports read with version. + pub read_with_version: bool, /// If operator supports write. pub write: bool, @@ -119,6 +123,8 @@ pub struct Capability { /// If operator supports delete. pub delete: bool, + /// if operator supports delete with version. + pub delete_with_version: bool, /// If operator supports copy. pub copy: bool, @@ -134,7 +140,7 @@ pub struct Capability { pub list_with_start_after: bool, /// If backend supports list with recursive. pub list_with_recursive: bool, - /// If backend supports list with object version. + /// if operator supports list with version. pub list_with_version: bool, /// If operator supports presign. diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 777c817e0f83..b5ddd1d4c32d 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -406,7 +406,7 @@ impl>> FutureWriter { /// /// ## Notes /// - /// we don't need to include the user defined metadata prefix in the key + /// we don't need to include the user defined metadata prefix in the key. /// every service will handle it internally pub fn user_metadata(self, data: impl IntoIterator) -> Self { self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options)) diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs index ab51712d696d..0225f50ab66e 100644 --- a/core/tests/behavior/async_delete.rs +++ b/core/tests/behavior/async_delete.rs @@ -33,7 +33,9 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_delete_with_special_chars, test_delete_not_existing, test_delete_stream, - test_remove_one_file + test_remove_one_file, + test_delete_with_version, + test_delete_with_not_existing_version )); if cap.list_with_recursive { tests.extend(async_trials!(op, test_remove_all_basic)); @@ -212,3 +214,70 @@ pub async fn test_remove_all_with_prefix_exists(op: Operator) -> Result<()> { .expect("write must succeed"); test_blocking_remove_all_with_objects(op, parent, ["a", "a/b", "a/c", "a/b/e"]).await } + +pub async fn test_delete_with_version(op: Operator) -> Result<()> { + if !op.info().full_capability().delete_with_version { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(path.as_str(), content) + .await + .expect("write must success"); + let meta = op.stat(path.as_str()).await.expect("stat must success"); + let version = meta.version().expect("must have version"); + + op.delete(path.as_str()).await.expect("delete must success"); + assert!(!op.is_exist(path.as_str()).await?); + + // After a simple delete, the data can still be accessed using its version. + let meta = op + .stat_with(path.as_str()) + .version(version) + .await + .expect("stat must success"); + assert_eq!(version, meta.version().expect("must have version")); + + // After deleting with the version, the data is removed permanently + op.delete_with(path.as_str()) + .version(version) + .await + .expect("delete must success"); + let ret = op.stat_with(path.as_str()).version(version).await; + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +pub async fn test_delete_with_not_existing_version(op: Operator) -> Result<()> { + if !op.info().full_capability().delete_with_version { + return Ok(()); + } + + // retrieve a valid version + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content) + .await + .expect("write must success"); + let version = op + .stat(path.as_str()) + .await + .expect("stat must success") + .version() + .expect("must have stat") + .to_string(); + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content) + .await + .expect("write must success"); + let ret = op + .delete_with(path.as_str()) + .version(version.as_str()) + .await; + assert!(ret.is_ok()); + + Ok(()) +} diff --git a/core/tests/behavior/async_read.rs b/core/tests/behavior/async_read.rs index 53e6f29a6102..d691165e5fba 100644 --- a/core/tests/behavior/async_read.rs +++ b/core/tests/behavior/async_read.rs @@ -44,7 +44,9 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_read_with_special_chars, test_read_with_override_cache_control, test_read_with_override_content_disposition, - test_read_with_override_content_type + test_read_with_override_content_type, + test_read_with_version, + test_read_with_not_existing_version )) } @@ -553,3 +555,66 @@ pub async fn test_read_only_read_with_if_none_match(op: Operator) -> anyhow::Res Ok(()) } + +pub async fn test_read_with_version(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_version { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let meta = op.stat(path.as_str()).await.expect("stat must success"); + let version = meta.version().expect("must have version"); + + let data = op + .read_with(path.as_str()) + .version(version) + .await + .expect("read must success"); + assert_eq!(content, data.to_vec()); + + op.write(path.as_str(), "1") + .await + .expect("write must success"); + + // After writing new data, we can still read the first version data + let second_data = op + .read_with(path.as_str()) + .version(version) + .await + .expect("read must success"); + assert_eq!(content, second_data.to_vec()); + + Ok(()) +} + +pub async fn test_read_with_not_existing_version(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_version { + return Ok(()); + } + + // retrieve a valid version + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let version = op + .stat(path.as_str()) + .await + .expect("stat must success") + .version() + .expect("must have version") + .to_string(); + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content) + .await + .expect("write must success"); + let ret = op.read_with(path.as_str()).version(&version).await; + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} diff --git a/core/tests/behavior/async_stat.rs b/core/tests/behavior/async_stat.rs index 2dcbde9b2dff..38f50704acd4 100644 --- a/core/tests/behavior/async_stat.rs +++ b/core/tests/behavior/async_stat.rs @@ -18,13 +18,12 @@ use std::str::FromStr; use std::time::Duration; +use crate::*; use anyhow::Result; use http::StatusCode; use log::warn; use reqwest::Url; -use crate::*; - pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); @@ -42,7 +41,9 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_stat_with_override_cache_control, test_stat_with_override_content_disposition, test_stat_with_override_content_type, - test_stat_root + test_stat_root, + test_stat_with_version, + stat_with_not_existing_version )) } @@ -166,12 +167,12 @@ pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> { pub async fn test_stat_not_exist(op: Operator) -> Result<()> { let path = uuid::Uuid::new_v4().to_string(); - // Stat not exist file should returns NotFound. + // Stat not exist file should return NotFound. let meta = op.stat(&path).await; assert!(meta.is_err()); assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - // Stat not exist dir should also returns NotFound. + // Stat not exist dir should also return NotFound. if op.info().full_capability().create_dir { let meta = op.stat(&format!("{path}/")).await; assert!(meta.is_err()); @@ -499,3 +500,70 @@ pub async fn test_read_only_stat_root(op: Operator) -> Result<()> { Ok(()) } + +pub async fn test_stat_with_version(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_version { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let first_meta = op.stat(path.as_str()).await.expect("stat must success"); + let first_version = first_meta.version().expect("must have version"); + + let first_versioning_meta = op + .stat_with(path.as_str()) + .version(first_version) + .await + .expect("stat must success"); + assert_eq!(first_meta, first_versioning_meta); + + op.write(path.as_str(), content) + .await + .expect("write must success"); + let second_meta = op.stat(path.as_str()).await.expect("stat must success"); + let second_version = second_meta.version().expect("must have version"); + assert_ne!(first_version, second_version); + + // we can still `stat` with first_version after writing new data + let meta = op + .stat_with(path.as_str()) + .version(first_version) + .await + .expect("stat must success"); + assert_eq!(first_meta, meta); + + Ok(()) +} + +pub async fn stat_with_not_existing_version(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_version { + return Ok(()); + } + + // retrieve a valid version + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let version = op + .stat(path.as_str()) + .await + .expect("stat must success") + .version() + .expect("must have version") + .to_string(); + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content) + .await + .expect("write must success"); + let ret = op.stat_with(path.as_str()).version(version.as_str()).await; + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +}