From 5d4300e0a27a4295b3b9fdcefd648f9b13eb8858 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 23 Sep 2023 15:04:26 +0800 Subject: [PATCH] test: resturcture java behavior tests (#3161) * test(bindings/java): polish tests and setup Signed-off-by: tison * docs: update Signed-off-by: tison * test: resturcture java behavior tests Signed-off-by: tison * tidy Signed-off-by: tison * fix s3 tests Signed-off-by: tison * fix jdk8 test compat Signed-off-by: tison * bump version Signed-off-by: tison * squash files Signed-off-by: tison --------- Signed-off-by: tison --- .github/workflows/bindings_java.yml | 3 +- .github/workflows/service_test_redis.yml | 4 +- .github/workflows/service_test_s3.yml | 29 +- bindings/java/README.md | 65 ++--- bindings/java/pom.xml | 41 +-- .../java/org/apache/opendal/Capability.java | 33 ++- .../org/apache/opendal/OperatorInfoTest.java | 86 +++--- .../java/org/apache/opendal/OperatorTest.java | 267 ------------------ .../behavior/AbstractBehaviorTest.java | 243 ++++++++++++++++ .../behavior/ServiceBehaviorTests.java | 87 ++++++ .../condition/OpenDALExceptionCondition.java | 5 + .../java/org/apache/opendal/utils/Utils.java | 161 ----------- .../test/resources/features/binding.feature | 1 - 13 files changed, 462 insertions(+), 563 deletions(-) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/OperatorTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/behavior/AbstractBehaviorTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/behavior/ServiceBehaviorTests.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/utils/Utils.java delete mode 120000 bindings/java/src/test/resources/features/binding.feature diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index 3aadb5a67869..343e0325e7b7 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -84,4 +84,5 @@ jobs: # https://maven.apache.org/guides/mini/guide-reproducible-builds.html#how-to-test-my-maven-build-reproducibility shell: bash run: | - ./mvnw clean install -DskipTests -Dcargo-build.features=services-redis + ./mvnw clean install -DskipTests + ./mvnw verify artifact:compare diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index 4b13ee770696..ccfba3671760 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -193,7 +193,7 @@ jobs: OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 - binding_java_redis: + java: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -209,7 +209,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -Dcargo-build.features=services-redis + run: ./mvnw test -Dtest=org.apache.opendal.behavior.RedisTest -Dcargo-build.features=services-redis env: OPENDAL_REDIS_TEST: on OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index a6ad92d0588b..15507b2bbe65 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -243,23 +243,30 @@ jobs: # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true - binding_java_s3: + java: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - + - name: Setup MinIO Server + shell: bash + working-directory: fixtures/s3 + run: docker-compose -f docker-compose-minio.yml up -d + - name: Setup test bucket + env: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + AWS_EC2_METADATA_DISABLED: "true" + run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - name: Setup Rust toolchain uses: ./.github/actions/setup - - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest + run: ./mvnw test -Dtest=org.apache.opendal.behavior.S3Test env: - OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} - OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} - OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} - OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} - OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }} - OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }} - OPENDAL_S3_REGION: ap-northeast-1 + OPENDAL_S3_TEST: on + OPENDAL_S3_BUCKET: test + OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" + OPENDAL_S3_ACCESS_KEY_ID: minioadmin + OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin + OPENDAL_S3_REGION: us-east-1 diff --git a/bindings/java/README.md b/bindings/java/README.md index ed8b8567b70b..48caad9dc899 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -15,19 +15,20 @@ Generally, you can first add the `os-maven-plugin` for automatically detect the ```xml - - - kr.motd.maven - os-maven-plugin - 1.7.0 - - + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + ``` Then add the dependency to `opendal-java` as following: ```xml + org.apache.opendal opendal-java @@ -39,32 +40,30 @@ Then add the dependency to `opendal-java` as following: ${opendal.version} ${os.detected.classifier} + ``` ### Gradle -For Gradle you can first add the `com.google.osdetector` for automatically detect the classifier based on your platform: +For Gradle, you can first add the `com.google.osdetector` for automatically detect the classifier based on your platform: ```groovy plugins { - ... id "com.google.osdetector" version "1.7.3" } - ``` Then add the dependency to `opendal-java` as following: ```groovy dependencies { - ... - // OpenDAL implementation "org.apache.opendal:opendal-java:0.40.0" implementation "org.apache.opendal:opendal-java:0.40.0:$osdetector.classifier" } ``` ### Classified library + Note that the dependency without classifier ships all classes and resources except the "opendal_java" shared library. And those with classifier bundle only the shared library. For downstream usage, it's recommended: @@ -98,20 +97,25 @@ You can run the base tests with the following command: ./mvnw clean verify ``` -## Run Service Tests +## Code style -Please copy `{project.rootdir}/.env.example` to `{project.rootdir}/.env` and change the values on need. +This project uses [spotless](https://github.com/diffplug/spotless) for code formatting so that all developers share a consistent code style without bikeshedding on it. -Take `fs` for example, we need to enable bench on `fs` on `/tmp`. +You can apply the code style with the following command:: -```dotenv -OPENDAL_FS_TEST=false -OPENDAL_FS_ROOT=/path/to/dir +```shell +./mvnw spotless:apply ``` -into +## Run Service Tests + +Services tests read necessary configs from env vars or the `.env` file. + +You can copy [.env.example](/.env.example) to `${project.rootdir}/.env` and change the values on need, or directly set env vars with `export KEY=VALUE`. -```dotenv +Take `fs` for example, we need to enable bench on `fs` on `/tmp`: + +```properties OPENDAL_FS_TEST=on OPENDAL_FS_ROOT=/opendal ``` @@ -119,26 +123,11 @@ OPENDAL_FS_ROOT=/opendal You can run service tests of enabled with the following command: ```shell -./mvnw test -Dtest=org.apache.opendal.OperatorTest +./mvnw test -Dtest=org.apache.opendal.behavior.FsTest # replace with the certain service tests ``` -You can run the unbound service with the following command: +Remember to enable the necessary features via `-Dcargo-build.features=services-xxx` when running specific service test: ```shell -./mvnw test -Dtest=org.apache.opendal.OperatorTest -Dcargo-build.features=services-redis -``` - -> **Note:** -> -> The `-Dcargo-build.features=services-redis` argument is a temporary workaround. See also: -> -> * https://github.com/apache/incubator-opendal/pull/3060 -> * https://github.com/apache/incubator-opendal/issues/3066 - -Additionally, this project uses [spotless](https://github.com/diffplug/spotless) for code formatting so that all developers share a consistent code style without bikeshedding on it. - -You can apply the code style with the following command:: - -```shell -./mvnw spotless:apply +./mvnw test -Dtest=org.apache.opendal.behavior.RedisTest -Dcargo-build.features=services-redis ``` diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml index 7d506382aad5..e4cbfbf05d00 100644 --- a/bindings/java/pom.xml +++ b/bindings/java/pom.xml @@ -59,12 +59,12 @@ 3.23.1 + 2.3.2 1.18.30 2.0.7 - 1.18.3 - 3.0.0 + 3.1.2 3.1.0 1.7.0 2.36.0 @@ -73,20 +73,6 @@ - - io.cucumber - cucumber-bom - 7.11.2 - pom - import - - - org.testcontainers - testcontainers-bom - ${testcontainers.version} - pom - import - org.junit junit-bom @@ -109,6 +95,11 @@ slf4j-simple ${slf4j.version} + + io.github.cdimascio + dotenv-java + ${dotenv.version} + @@ -119,16 +110,6 @@ provided - - io.cucumber - cucumber-java - test - - - io.cucumber - cucumber-junit-platform-engine - test - org.junit.platform junit-platform-suite @@ -145,13 +126,13 @@ test - org.testcontainers - junit-jupiter + org.slf4j + slf4j-simple test - org.slf4j - slf4j-simple + io.github.cdimascio + dotenv-java test diff --git a/bindings/java/src/main/java/org/apache/opendal/Capability.java b/bindings/java/src/main/java/org/apache/opendal/Capability.java index 2291738bb38f..ad8a884e1738 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Capability.java +++ b/bindings/java/src/main/java/org/apache/opendal/Capability.java @@ -19,18 +19,22 @@ package org.apache.opendal; +import lombok.EqualsAndHashCode; import lombok.ToString; @ToString +@EqualsAndHashCode public class Capability { /** * If operator supports stat. */ public final boolean stat; + /** - * If operator supports stat with if match. + * If operator supports stat with if matched. */ public final boolean statWithIfMatch; + /** * If operator supports stat with if none match. */ @@ -40,34 +44,42 @@ public class Capability { * If operator supports read. */ public final boolean read; + /** * If operator supports seek on returning reader. */ public final boolean readCanSeek; + /** * If operator supports next on returning reader. */ public final boolean readCanNext; + /** * If operator supports read with range. */ public final boolean readWithRange; + /** - * If operator supports read with if match. + * If operator supports read with if matched. */ public final boolean readWithIfMatch; + /** * If operator supports read with if none match. */ public final boolean readWithIfNoneMatch; + /** * If operator supports read with override cache control. */ public final boolean readWithOverrideCacheControl; + /** * if operator supports read with override content disposition. */ public final boolean readWithOverrideContentDisposition; + /** * if operator supports read with override content type. */ @@ -77,36 +89,44 @@ public class Capability { * If operator supports write. */ public final boolean write; + /** * If operator supports write can be called in multi times. */ public final boolean writeCanMulti; + /** * If operator supports write by append. */ public final boolean writeCanAppend; + /** * If operator supports write with content type. */ public final boolean writeWithContentType; + /** * If operator supports write with content disposition. */ public final boolean writeWithContentDisposition; + /** * If operator supports write with cache control. */ public final boolean writeWithCacheControl; + /** * write_multi_max_size is the max size that services support in write_multi. * For example, AWS S3 supports 5GiB as max in write_multi. */ public final long writeMultiMaxSize; + /** * write_multi_min_size is the min size that services support in write_multi. * For example, AWS S3 requires at least 5MiB in write_multi expect the last one. */ public final long writeMultiMinSize; + /** * write_multi_align_size is the align size that services required in write_multi. * For example, Google GCS requires align size to 256KiB in write_multi. @@ -137,18 +157,22 @@ public class Capability { * If operator supports list. */ public final boolean list; + /** * If backend supports list with limit. */ public final boolean listWithLimit; + /** * If backend supports list with start after. */ public final boolean listWithStartAfter; + /** * If backend support list with using slash as delimiter. */ public final boolean listWithDelimiterSlash; + /** * If backend supports list without delimiter. */ @@ -158,14 +182,17 @@ public class Capability { * If operator supports presign. */ public final boolean presign; + /** * If operator supports presign read. */ public final boolean presignRead; + /** * If operator supports presign stat. */ public final boolean presignStat; + /** * If operator supports presign write. */ @@ -175,10 +202,12 @@ public class Capability { * If operator supports batch. */ public final boolean batch; + /** * If operator supports batch delete. */ public final boolean batchDelete; + /** * The max operations that operator supports in batch. */ diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java index e4ea7786a217..7522d73e904b 100644 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java @@ -19,9 +19,7 @@ package org.apache.opendal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -29,65 +27,53 @@ import org.junit.jupiter.api.io.TempDir; public class OperatorInfoTest { - @TempDir private static Path tempDir; @Test public void testBlockingOperatorInfo() { - Map conf = new HashMap<>(); + final Map conf = new HashMap<>(); conf.put("root", tempDir.toString()); - try (BlockingOperator op = new BlockingOperator("fs", conf)) { - - OperatorInfo info = op.info(); - assertNotNull(info); - assertEquals("fs", info.scheme); - - Capability fullCapability = info.fullCapability; - assertNotNull(fullCapability); - - assertTrue(fullCapability.read); - assertTrue(fullCapability.write); - assertTrue(fullCapability.delete); - assertTrue(fullCapability.writeCanAppend); - - assertEquals(fullCapability.writeMultiAlignSize, -1); - assertEquals(fullCapability.writeMultiMaxSize, -1); - assertEquals(fullCapability.writeMultiMinSize, -1); - assertEquals(fullCapability.batchMaxOperations, -1); - Capability nativeCapability = info.nativeCapability; - assertNotNull(nativeCapability); + try (final BlockingOperator op = new BlockingOperator("fs", conf)) { + final OperatorInfo info = op.info(); + assertThat(info).isNotNull(); + assertThat(info.scheme).isEqualTo("fs"); + + assertThat(info.fullCapability).isNotNull(); + assertThat(info.fullCapability.read).isTrue(); + assertThat(info.fullCapability.write).isTrue(); + assertThat(info.fullCapability.delete).isTrue(); + assertThat(info.fullCapability.writeCanAppend).isTrue(); + assertThat(info.fullCapability.writeMultiAlignSize).isEqualTo(-1); + assertThat(info.fullCapability.writeMultiMaxSize).isEqualTo(-1); + assertThat(info.fullCapability.writeMultiMinSize).isEqualTo(-1); + assertThat(info.fullCapability.batchMaxOperations).isEqualTo(-1); + + assertThat(info.nativeCapability).isNotNull(); } } @Test public void testOperatorInfo() { - Map conf = new HashMap<>(); - String root = "/opendal/"; - conf.put("root", root); - try (Operator op = new Operator("memory", conf)) { - - OperatorInfo info = op.info(); - assertNotNull(info); - assertEquals("memory", info.scheme); - assertEquals(root, info.root); - - Capability fullCapability = info.fullCapability; - assertNotNull(fullCapability); - - assertTrue(fullCapability.read); - assertTrue(fullCapability.write); - assertTrue(fullCapability.delete); - assertTrue(!fullCapability.writeCanAppend); - - assertEquals(fullCapability.writeMultiAlignSize, -1); - assertEquals(fullCapability.writeMultiMaxSize, -1); - assertEquals(fullCapability.writeMultiMinSize, -1); - assertEquals(fullCapability.batchMaxOperations, -1); - - Capability nativeCapability = info.nativeCapability; - assertNotNull(nativeCapability); + final Map conf = new HashMap<>(); + conf.put("root", "/opendal/"); + try (final Operator op = new Operator("memory", conf)) { + final OperatorInfo info = op.info(); + assertThat(info).isNotNull(); + assertThat(info.scheme).isEqualTo("memory"); + + assertThat(info.fullCapability).isNotNull(); + assertThat(info.fullCapability.read).isTrue(); + assertThat(info.fullCapability.write).isTrue(); + assertThat(info.fullCapability.delete).isTrue(); + assertThat(info.fullCapability.writeCanAppend).isFalse(); + assertThat(info.fullCapability.writeMultiAlignSize).isEqualTo(-1); + assertThat(info.fullCapability.writeMultiMaxSize).isEqualTo(-1); + assertThat(info.fullCapability.writeMultiMinSize).isEqualTo(-1); + assertThat(info.fullCapability.batchMaxOperations).isEqualTo(-1); + + assertThat(info.nativeCapability).isNotNull(); } } } diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java deleted file mode 100644 index f77529cdd793..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.apache.opendal.utils.Utils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -public class OperatorTest { - - private static final List ops = new ArrayList<>(); - - private static final List blockingOps = new ArrayList<>(); - - protected static final String[] schemas = new String[] { - "atomicserver", - "azblob", - "azdls", - "cacache", - "cos", - "dashmap", - "etcd", - "foundationdb", - "fs", - "ftp", - "gcs", - "ghac", - "hdfs", - "http", - "ipfs", - "ipmfs", - "memcached", - "memory", - "minimoka", - "moka", - "obs", - "onedrive", - "gdrive", - "dropbox", - "oss", - "persy", - "redis", - "postgresql", - "rocksdb", - "s3", - "sftp", - "sled", - "supabase", - "vercel-artifacts", - "wasabi", - "webdav", - "webhdfs", - "redb", - "tikv", - }; - - @BeforeAll - public static void init() { - for (String schema : schemas) { - - Optional opOptional = Utils.init(schema); - opOptional.ifPresent(op -> ops.add(op)); - - Optional blockingOpOptional = Utils.initBlockingOp(schema); - blockingOpOptional.ifPresent(op -> blockingOps.add(op)); - } - if (ops.isEmpty()) { - ops.add(null); - } - if (blockingOps.isEmpty()) { - blockingOps.add(null); - } - } - - @AfterAll - public static void clean() { - ops.stream().filter(Objects::nonNull).forEach(Operator::close); - blockingOps.stream().filter(Objects::nonNull).forEach(BlockingOperator::close); - } - - private static Stream getOperators() { - return ops.stream(); - } - - private static Stream getBlockingOperators() { - return blockingOps.stream(); - } - - @ParameterizedTest(autoCloseArguments = false) - @MethodSource("getBlockingOperators") - public void testBlockingWrite(BlockingOperator blockingOp) { - assumeTrue(blockingOp != null); - - Capability cap = blockingOp.info().fullCapability; - if (!cap.write || !cap.read) { - return; - } - - String path = UUID.randomUUID().toString(); - byte[] content = Utils.generateBytes(); - blockingOp.write(path, content); - - Metadata metadata = blockingOp.stat(path); - - assertEquals(content.length, metadata.getContentLength()); - - blockingOp.delete(path); - assertThatThrownBy(() -> blockingOp.stat(path)) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @ParameterizedTest(autoCloseArguments = false) - @MethodSource("getBlockingOperators") - public void testBlockingRead(BlockingOperator blockingOp) { - assumeTrue(blockingOp != null); - - Capability cap = blockingOp.info().fullCapability; - if (!cap.write || !cap.read) { - return; - } - - Metadata metadata = blockingOp.stat(""); - assertTrue(!metadata.isFile()); - - String path = UUID.randomUUID().toString(); - assertThatThrownBy(() -> blockingOp.stat(path)) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - - byte[] content = Utils.generateBytes(); - blockingOp.write(path, content); - - assertThat(blockingOp.read(path)).isEqualTo(content); - - blockingOp.delete(path); - assertThatThrownBy(() -> blockingOp.stat(path)) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @ParameterizedTest(autoCloseArguments = false) - @MethodSource("getOperators") - public final void testWrite(Operator op) throws Exception { - assumeTrue(op != null); - - Capability cap = op.info().fullCapability; - if (!cap.write || !cap.read) { - return; - } - - String path = UUID.randomUUID().toString(); - byte[] content = Utils.generateBytes(); - op.write(path, content).join(); - - Metadata metadata = op.stat(path).get(); - - assertEquals(content.length, metadata.getContentLength()); - - op.delete(path).join(); - assertThatThrownBy(() -> op.stat(path).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @ParameterizedTest(autoCloseArguments = false) - @MethodSource("getOperators") - public final void testRead(Operator op) throws Exception { - assumeTrue(op != null); - - Capability cap = op.info().fullCapability; - if (!cap.write || !cap.read) { - return; - } - - Metadata metadata = op.stat("").get(); - assertTrue(!metadata.isFile()); - - String path = UUID.randomUUID().toString(); - assertThatThrownBy(() -> op.stat(path).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - - byte[] content = Utils.generateBytes(); - op.write(path, content).join(); - - assertThat(op.read(path).join()).isEqualTo(content); - - op.delete(path).join(); - assertThatThrownBy(() -> op.stat(path).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @ParameterizedTest(autoCloseArguments = false) - @MethodSource("getOperators") - public void testAppend(Operator op) { - assumeTrue(op != null); - - Capability cap = op.info().fullCapability; - if (!cap.write || !cap.writeCanAppend || !cap.read) { - return; - } - - String path = UUID.randomUUID().toString(); - byte[][] trunks = new byte[][] {Utils.generateBytes(), Utils.generateBytes(), Utils.generateBytes()}; - - for (int i = 0; i < trunks.length; i++) { - op.append(path, trunks[i]).join(); - - byte[] expected = Arrays.stream(trunks).limit(i + 1).reduce(new byte[0], (arr1, arr2) -> { - byte[] result = new byte[arr1.length + arr2.length]; - System.arraycopy(arr1, 0, result, 0, arr1.length); - System.arraycopy(arr2, 0, result, arr1.length, arr2.length); - return result; - }); - - assertThat(op.read(path).join()).isEqualTo(expected); - } - - // write overwrite existing content - byte[] newAttempt = Utils.generateBytes(); - op.write(path, newAttempt).join(); - assertThat(op.read(path).join()).isEqualTo(newAttempt); - - for (int i = 0; i < trunks.length; i++) { - op.append(path, trunks[i]).join(); - - byte[] expected = Stream.concat( - Stream.of(newAttempt), Arrays.stream(trunks).limit(i + 1)) - .reduce(new byte[0], (arr1, arr2) -> { - byte[] result = new byte[arr1.length + arr2.length]; - System.arraycopy(arr1, 0, result, 0, arr1.length); - System.arraycopy(arr2, 0, result, arr1.length, arr2.length); - return result; - }); - - assertThat(op.read(path).join()).isEqualTo(expected); - } - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/behavior/AbstractBehaviorTest.java b/bindings/java/src/test/java/org/apache/opendal/behavior/AbstractBehaviorTest.java new file mode 100644 index 000000000000..4fad3b065c74 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/behavior/AbstractBehaviorTest.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.behavior; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvEntry; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import org.apache.opendal.BlockingOperator; +import org.apache.opendal.Capability; +import org.apache.opendal.Metadata; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.Operator; +import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class AbstractBehaviorTest { + protected final String scheme; + protected final Map config; + protected Operator operator; + protected BlockingOperator blockingOperator; + + protected AbstractBehaviorTest(String scheme) { + this(scheme, createSchemeConfig(scheme)); + } + + protected AbstractBehaviorTest(String scheme, Map config) { + this.scheme = scheme; + this.config = config; + } + + @BeforeAll + public void setup() { + assertThat(isSchemeEnabled(config)) + .describedAs("service test for " + scheme + " is not enabled.") + .isTrue(); + this.operator = new Operator(scheme, config); + this.blockingOperator = new BlockingOperator(scheme, config); + } + + @AfterAll + public void teardown() { + if (operator != null) { + operator.close(); + operator = null; + } + if (blockingOperator != null) { + blockingOperator.close(); + blockingOperator = null; + } + } + + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @Nested + class AsyncWriteTest { + @BeforeAll + public void precondition() { + final Capability capability = operator.info().fullCapability; + assumeTrue(capability.read && capability.write); + } + + /** + * Read not exist file should return NotFound. + */ + @Test + public void testReadNotExist() { + final String path = UUID.randomUUID().toString(); + assertThatThrownBy(() -> operator.read(path).join()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + + /** + * Read full content should match. + */ + @Test + public void testReadFull() { + final String path = UUID.randomUUID().toString(); + final byte[] content = generateBytes(); + operator.write(path, content).join(); + final byte[] actualContent = operator.read(path).join(); + assertThat(actualContent).isEqualTo(content); + operator.delete(path).join(); + } + + /** + * Stat not exist file should return NotFound. + */ + @Test + public void testStatNotExist() { + final String path = UUID.randomUUID().toString(); + assertThatThrownBy(() -> operator.stat(path).join()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + + /** + * Stat existing file should return metadata. + */ + @Test + public void testStatFile() { + final String path = UUID.randomUUID().toString(); + final byte[] content = generateBytes(); + operator.write(path, content).join(); + try (final Metadata meta = operator.stat(path).join()) { + assertThat(meta.isFile()).isTrue(); + assertThat(meta.getContentLength()).isEqualTo(content.length); + } + operator.delete(path).join(); + } + } + + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @Nested + class AsyncAppendTest { + @BeforeAll + public void precondition() { + final Capability capability = operator.info().fullCapability; + assumeTrue(capability.read && capability.write && capability.writeCanAppend); + } + + @Test + public void testAppendCreateAppend() { + final String path = UUID.randomUUID().toString(); + final byte[] contentOne = generateBytes(); + final byte[] contentTwo = generateBytes(); + + operator.append(path, contentOne).join(); + operator.append(path, contentTwo).join(); + + final byte[] actualContent = operator.read(path).join(); + assertThat(actualContent.length).isEqualTo(contentOne.length + contentTwo.length); + assertThat(Arrays.copyOfRange(actualContent, 0, contentOne.length)).isEqualTo(contentOne); + assertThat(Arrays.copyOfRange(actualContent, contentOne.length, actualContent.length)) + .isEqualTo(contentTwo); + + operator.delete(path).join(); + } + } + + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @Nested + class BlockingWriteTest { + @BeforeAll + public void precondition() { + final Capability capability = blockingOperator.info().fullCapability; + assumeTrue(capability.read && capability.write && capability.blocking); + } + + /** + * Read not exist file should return NotFound. + */ + @Test + public void testBlockingReadNotExist() { + final String path = UUID.randomUUID().toString(); + assertThatThrownBy(() -> blockingOperator.read(path)) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.NotFound)); + } + + /** + * Read full content should match. + */ + @Test + public void testBlockingReadFull() { + final String path = UUID.randomUUID().toString(); + final byte[] content = generateBytes(); + blockingOperator.write(path, content); + final byte[] actualContent = blockingOperator.read(path); + assertThat(actualContent).isEqualTo(content); + blockingOperator.delete(path); + } + + /** + * Stat existing file should return metadata. + */ + @Test + public void testBlockingStatFile() { + final String path = UUID.randomUUID().toString(); + final byte[] content = generateBytes(); + blockingOperator.write(path, content); + try (final Metadata meta = blockingOperator.stat(path)) { + assertThat(meta.isFile()).isTrue(); + assertThat(meta.getContentLength()).isEqualTo(content.length); + } + blockingOperator.delete(path); + } + } + + /** + * Generates a byte array of random content. + */ + public static byte[] generateBytes() { + final Random random = new Random(); + final int size = random.nextInt(4 * 1024 * 1024) + 1; + final byte[] content = new byte[size]; + random.nextBytes(content); + return content; + } + + protected static boolean isSchemeEnabled(Map config) { + final String turnOn = config.getOrDefault("test", "").toLowerCase(); + return turnOn.equals("on") || turnOn.equals("true"); + } + + protected static Map createSchemeConfig(String scheme) { + final Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); + final Map config = new HashMap<>(); + final String prefix = "opendal_" + scheme.toLowerCase() + "_"; + for (DotenvEntry entry : dotenv.entries()) { + final String key = entry.getKey().toLowerCase(); + if (key.startsWith(prefix)) { + config.put(key.substring(prefix.length()), entry.getValue()); + } + } + return config; + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/behavior/ServiceBehaviorTests.java b/bindings/java/src/test/java/org/apache/opendal/behavior/ServiceBehaviorTests.java new file mode 100644 index 000000000000..9615f97c97fd --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/behavior/ServiceBehaviorTests.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.behavior; + +import java.io.File; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.util.Files; +import org.junit.jupiter.api.condition.EnabledIf; + +@Slf4j +class MemoryTest extends AbstractBehaviorTest { + public MemoryTest() { + super("memory", defaultSchemeConfig()); + } + + private static Map defaultSchemeConfig() { + final Map config = createSchemeConfig("memory"); + if (!isSchemeEnabled(config)) { + log.info("Running MemoryTest with default config."); + config.clear(); + config.put("test", "on"); + config.put("root", "/tmp"); + } + return config; + } +} + +@Slf4j +class FsTest extends AbstractBehaviorTest { + public FsTest() { + super("fs", schemeConfig()); + } + + private static Map schemeConfig() { + final Map config = createSchemeConfig("fs"); + if (!isSchemeEnabled(config)) { + log.info("Running FsTest with default config."); + config.clear(); + + final File tempDir = Files.newTemporaryFolder(); + tempDir.deleteOnExit(); + config.put("test", "on"); + config.put("root", tempDir.getAbsolutePath()); + } + return config; + } +} + +@EnabledIf("enabled") +class RedisTest extends AbstractBehaviorTest { + public RedisTest() { + super("redis"); + } + + private static boolean enabled() { + return isSchemeEnabled(createSchemeConfig("redis")); + } +} + +@EnabledIf("enabled") +class S3Test extends AbstractBehaviorTest { + public S3Test() { + super("s3"); + } + + private static boolean enabled() { + return isSchemeEnabled(createSchemeConfig("s3")); + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java b/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java index 4c3afddb8b48..c1595f23946a 100644 --- a/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java +++ b/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java @@ -20,6 +20,7 @@ package org.apache.opendal.condition; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -38,6 +39,10 @@ public static OpenDALExceptionCondition ofAsync(OpenDALException.Code code) { return new OpenDALExceptionCondition(code, canBeStripped); } + public static OpenDALExceptionCondition ofSync(OpenDALException.Code code) { + return new OpenDALExceptionCondition(code, Collections.emptyList()); + } + public OpenDALExceptionCondition(OpenDALException.Code code, List> canBeStripped) { as("OpenDALException with code " + code + ", stripping " + canBeStripped); this.code = code; diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java deleted file mode 100644 index 0121002d4973..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.utils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.Random; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.opendal.BlockingOperator; -import org.apache.opendal.Operator; - -public class Utils { - - public static final String ENV_NAME = ".env"; - public static final String CONF_PREFIX = "opendal_"; - public static final String CONF_TURN_ON_TEST = "test"; - public static final String CONF_ROOT = "root"; - public static final String CONF_RANDOM_ROOT_FLAG = "OPENDAL_DISABLE_RANDOM_ROOT"; - - /** - * Initializes the Service with the given schema. - * - * @param schema the schema to initialize the Operator service - * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. - * Else, returns a `Empty` to represent no valid config for operator. - */ - public static Optional init(String schema) { - Map conf = readEnv(schema); - - final String turnOnTest = conf.get(CONF_TURN_ON_TEST); - if (!isTurnOn(turnOnTest)) { - return Optional.empty(); - } - Operator op = new Operator(schema, conf); - return Optional.of(op); - } - - /** - * Initializes a blocking operator using the provided schema. - * - * @param schema the schema to be used for initializing the blocking operator - * @return If `opendal_{schema}_test` is on, construct a new BlockingOperator with given root. - * Else, returns a `Empty` to represent no valid config for operator. - */ - public static Optional initBlockingOp(String schema) { - Map conf = readEnv(schema); - - final String turnOnTest = conf.get(CONF_TURN_ON_TEST); - if (!isTurnOn(turnOnTest)) { - return Optional.empty(); - } - BlockingOperator op = new BlockingOperator(schema, conf); - return Optional.of(op); - } - - /** - * Reads the environment variables and system properties and returns a map - * containing the configuration settings for the given schema. - * - * @param schema the schema for which to retrieve the configuration settings - * @return a map containing the configuration settings - */ - private static Map readEnv(String schema) { - final Properties properties = new Properties(); - - String projectRoot = System.getProperty("user.dir"); - - projectRoot = Optional.ofNullable(Paths.get(projectRoot)) - .map(Path::getParent) - .map(Path::getParent) - .map(Path::toString) - .orElse(projectRoot); - - try (BufferedReader reader = - new BufferedReader(new FileReader(projectRoot.toString() + File.separator + ENV_NAME))) { - properties.load(reader); - } catch (Exception ignore) { - } - for (Map.Entry entry : System.getenv().entrySet()) { - properties.setProperty(entry.getKey(), entry.getValue()); - } - - final String confPrefix = (CONF_PREFIX + schema).toLowerCase(); - final Map conf = properties.entrySet().stream() - .filter(Objects::nonNull) - .filter(entry -> Optional.ofNullable(entry.getKey()) - .map(Object::toString) - .orElse("") - .toLowerCase() - .startsWith(confPrefix)) - .collect(Collectors.toMap( - entry -> { - String key = entry.getKey().toString().toLowerCase(); - return key.replace(confPrefix + "_", ""); - }, - entry -> Optional.ofNullable(entry.getValue()) - .map(Object::toString) - .orElse(""), - (existing, replacement) -> existing)); - - if (!Boolean.parseBoolean(properties.getProperty(CONF_RANDOM_ROOT_FLAG))) { - String root = conf.getOrDefault(CONF_ROOT, File.separator); - if (!root.endsWith(File.separator)) { - root = root + File.separator; - } - root = root + UUID.randomUUID() + File.separator; - conf.put(CONF_ROOT, root); - } - return conf; - } - - /** - * Determines if the given value is turn on. - * - * @param val the value to be checked - * @return true if the value is "on" or "true", false otherwise - */ - public static boolean isTurnOn(String val) { - return "on".equalsIgnoreCase(val) || "true".equalsIgnoreCase(val); - } - - /** - * Generates a byte array of random content. - * - * @return the generated byte array - */ - public static byte[] generateBytes() { - Random random = new Random(); - - int size = random.nextInt(4 * 1024 * 1024) + 1; - byte[] content = new byte[size]; - random.nextBytes(content); - - return content; - } -} diff --git a/bindings/java/src/test/resources/features/binding.feature b/bindings/java/src/test/resources/features/binding.feature deleted file mode 120000 index ad3840b3e359..000000000000 --- a/bindings/java/src/test/resources/features/binding.feature +++ /dev/null @@ -1 +0,0 @@ -../../../../../tests/features/binding.feature \ No newline at end of file