From a0a917f66ddb44a25259197c7e36051d4f8fd436 Mon Sep 17 00:00:00 2001 From: Suyan Date: Wed, 24 Jan 2024 13:26:09 +0800 Subject: [PATCH] feat(services/koofr): set test for koofr (#4050) * feat(services/koofr): set test for koofr Signed-off-by: suyanhanx * fix env var name Signed-off-by: suyanhanx * fix feature flag in bindings Signed-off-by: suyanhanx * fix setup config Signed-off-by: suyanhanx * fix skip condition Signed-off-by: suyanhanx * fix skip condition Signed-off-by: suyanhanx * fix format Signed-off-by: suyanhanx * remove test-threads limit Signed-off-by: suyanhanx --------- Signed-off-by: suyanhanx --- .env.example | 4 +-- .github/services/koofr/koofr/action.yml | 32 +++++++++++++++++++ bindings/java/Cargo.toml | 2 ++ .../opendal/test/behavior/AsyncCopyTest.java | 4 +++ bindings/nodejs/Cargo.toml | 18 ++++++----- bindings/python/Cargo.toml | 4 ++- core/Cargo.toml | 2 +- core/src/services/koofr/backend.rs | 26 ++++++++++++--- core/tests/behavior/async_copy.rs | 5 +++ 9 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 .github/services/koofr/koofr/action.yml diff --git a/.env.example b/.env.example index ec9a1292b242..bcb6952d97de 100644 --- a/.env.example +++ b/.env.example @@ -205,10 +205,10 @@ OPENDAL_PCLOUD_PASSWORD= OPENDAL_YANDEX_DISK_ROOT=/path/to/dir OPENDAL_YANDEX_DISK_ACCESS_TOKEN= # koofr -OPENDAL_KOOFR_DISK_ROOT=/path/to/dir +OPENDAL_KOOFR_ROOT=/path/to/dir OPENDAL_KOOFR_ENDPOINT= OPENDAL_KOOFR_EMAIL= -OPENDAL_KOOFR_PASSWORD=> +OPENDAL_KOOFR_PASSWORD= # icloud OPENDAL_ICLOUD_ROOT=/ OPENDAL_ICLOUD_APPLE_ID= diff --git a/.github/services/koofr/koofr/action.yml b/.github/services/koofr/koofr/action.yml new file mode 100644 index 000000000000..20dd0bcd91b6 --- /dev/null +++ b/.github/services/koofr/koofr/action.yml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: koofr +description: 'Behavior test for Koofr. This service is sponsored by @suyanhanx.' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_KOOFR_ROOT: op://services/koofr/root + OPENDAL_KOOFR_ENDPOINT: op://services/koofr/endpoint + OPENDAL_KOOFR_EMAIL: op://services/koofr/email + OPENDAL_KOOFR_PASSWORD: op://services/koofr/password diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 9787a7325f9b..12bff79f6435 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -72,6 +72,7 @@ services-all = [ "services-onedrive", "services-persy", "services-postgresql", + "services-koofr", "services-mysql", "services-redb", "services-redis", @@ -125,6 +126,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java index b292e30c42ff..5944c02ea94b 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncCopyTest.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.util.Objects; import java.util.UUID; import org.apache.opendal.Capability; import org.apache.opendal.OpenDALException; @@ -64,6 +65,9 @@ public void testCopyFileWithAsciiName() { */ @Test public void testCopyFileWithNonAsciiName() { + // Services-koofr doesn't support non-ascii name (https://github.com/apache/opendal/issues/4051) + assumeTrue(!Objects.equals(op().info.scheme, "koofr"), "Services-koofr doesn't support non-ascii name"); + final String sourcePath = "🐂🍺中文.docx"; final String targetPath = "😈🐅Français.docx"; final byte[] content = generateBytes(); diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index 51da25bd668a..937a227ce07e 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -49,7 +49,9 @@ default = [ services-all = [ "default", + "services-alluxio", "services-azfile", + "services-b2", "services-cacache", "services-dashmap", "services-dropbox", @@ -59,13 +61,17 @@ services-all = [ # FIXME waiting for this issue close https://github.com/async-rs/async-tls/issues/55 # "services-ftp", "services-gdrive", + "services-gridfs", # FIXME how to support HDFS services in bindings? # "services-hdfs", "services-huggingface", "services-ipfs", + "services-koofr", + "services-libsql", "services-memcached", "services-mini-moka", "services-moka", + "services-mongodb", "services-onedrive", "services-persy", "services-postgresql", @@ -73,20 +79,15 @@ services-all = [ "services-redb", "services-redis", "services-rocksdb", + "services-seafile", "services-sled", + "services-sqlite", "services-supabase", "services-swift", "services-tikv", + "services-upyun", "services-vercel-artifacts", "services-wasabi", - "services-mongodb", - "services-gridfs", - "services-sqlite", - "services-libsql", - "services-alluxio", - "services-b2", - "services-seafile", - "services-upyun", ] # Default services provided by opendal. @@ -120,6 +121,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index fe00e54af13a..d065e1b39b44 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -89,6 +89,7 @@ services-all = [ "services-b2", "services-seafile", "services-upyun", + "services-koofr", ] # Default services provided by opendal. @@ -122,6 +123,7 @@ services-gridfs = ["opendal/services-gridfs"] services-hdfs = ["opendal/services-hdfs"] services-huggingface = ["opendal/services-huggingface"] services-ipfs = ["opendal/services-ipfs"] +services-koofr = ["opendal/services-koofr"] services-libsql = ["opendal/services-libsql"] services-memcached = ["opendal/services-memcached"] services-mini-moka = ["opendal/services-mini-moka"] @@ -161,4 +163,4 @@ features = [ # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. "services-sftp", ] -workspace = true \ No newline at end of file +workspace = true diff --git a/core/Cargo.toml b/core/Cargo.toml index 276aa7f2f179..eeec171d907f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -197,7 +197,7 @@ services-upyun = ["dep:hmac", "dep:sha1"] services-vercel-artifacts = [] # Deprecated # wasabi services support has been removed. -# We will remove this feature in next version. +# We will remove this feature in the next version. services-wasabi = [] services-webdav = [] services-webhdfs = [] diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index 6e19d9a7e78e..a7f8fc35901c 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -50,7 +50,7 @@ pub struct KoofrConfig { pub endpoint: String, /// Koofr email. pub email: String, - /// password of this backend. + /// password of this backend. (Must be the application password) pub password: Option, } @@ -115,10 +115,16 @@ impl KoofrBuilder { self } - /// Koofr app password. + /// Koofr application password. /// /// Go to https://app.koofr.net/app/admin/preferences/password. - /// Click "Generate Password" button to generate a new password. + /// Click "Generate Password" button to generate a new application password. + /// + /// # Notes + /// + /// This is not user's Koofr account password. + /// Please use the application password instead. + /// Please also remind users of this. pub fn password(&mut self, password: &str) -> &mut Self { self.config.password = if password.is_empty() { None @@ -173,7 +179,6 @@ impl Builder for KoofrBuilder { let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", &root); - // Handle endpoint. if self.config.endpoint.is_empty() { return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -182,7 +187,6 @@ impl Builder for KoofrBuilder { debug!("backend use endpoint {}", &self.config.endpoint); - // Handle email. if self.config.email.is_empty() { return Err(Error::new(ErrorKind::ConfigInvalid, "email is empty") .with_operation("Builder::build") @@ -253,7 +257,9 @@ impl Accessor for KoofrBackend { write_can_empty: true, delete: true, + rename: true, + copy: true, list: true, @@ -348,6 +354,11 @@ impl Accessor for KoofrBackend { async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result { self.core.ensure_dir_exists(to).await?; + + if from == to { + return Ok(RpCopy::default()); + } + let resp = self.core.remove(to).await?; let status = resp.status(); @@ -372,6 +383,11 @@ impl Accessor for KoofrBackend { async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result { self.core.ensure_dir_exists(to).await?; + + if from == to { + return Ok(RpRename::default()); + } + let resp = self.core.remove(to).await?; let status = resp.status(); diff --git a/core/tests/behavior/async_copy.rs b/core/tests/behavior/async_copy.rs index cee9494ec8a4..82dc4637ed8e 100644 --- a/core/tests/behavior/async_copy.rs +++ b/core/tests/behavior/async_copy.rs @@ -63,6 +63,11 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> { /// Copy a file with non ascii name and test contents. pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> { + // Koofr does not support non-ascii name.(https://github.com/apache/opendal/issues/4051) + if op.info().scheme() == Scheme::Koofr { + return Ok(()); + } + let source_path = "🐂🍺中文.docx"; let target_path = "😈🐅Français.docx"; let (source_content, _) = gen_bytes(op.info().full_capability());