diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts index 29676d8fa22..e0d8e9e15a9 100644 --- a/bundles/aliyun-bundle/build.gradle.kts +++ b/bundles/aliyun-bundle/build.gradle.kts @@ -29,12 +29,12 @@ dependencies { compileOnly(libs.hadoop3.common) implementation(libs.hadoop3.oss) - // oss needs StringUtils from commons-lang or the following error will occur in 3.1.0 - // java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils + // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 + // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) - implementation(libs.commons.lang) + implementation(libs.commons.lang3) implementation(project(":catalogs:catalog-common")) { exclude("*") } @@ -48,7 +48,7 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("org.jdom", "org.apache.gravitino.shaded.org.jdom") - relocate("org.apache.commons.lang", "org.apache.gravitino.shaded.org.apache.commons.lang") + relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3") } tasks.jar { diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java index 2e52e74bba7..da056f20d88 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -36,18 +37,14 @@ import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; -@Tag("gravitino-docker-test") -@Disabled( - "Disabled due to we don't have a real GCP account to test. If you have a GCP account," - + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") +@EnabledIf(value = "isGCPConfigured", disabledReason = "GCP is not configured.") public class HadoopGCSCatalogIT extends HadoopCatalogIT { - public static final String BUCKET_NAME = "YOUR_BUCKET"; - public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + public static final String BUCKET_NAME = System.getenv("GCS_BUCKET_NAME"); + public static final String SERVICE_ACCOUNT_FILE = System.getenv("GCS_SERVICE_ACCOUNT_JSON_PATH"); @Override public void startIntegrationTest() throws Exception { @@ -172,4 +169,9 @@ public void testCreateSchemaAndFilesetWithSpecialLocation() { // Delete catalog metalake.dropCatalog(localCatalogName, true); } + + private static boolean isGCPConfigured() { + return StringUtils.isNotBlank(System.getenv("GCS_SERVICE_ACCOUNT_JSON_PATH")) + && StringUtils.isNotBlank(System.getenv("GCS_BUCKET_NAME")); + } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopOSSCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopOSSCatalogIT.java index b6c3ac7224a..123e4226f19 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopOSSCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopOSSCatalogIT.java @@ -38,21 +38,19 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Disabled( - "Disabled due to we don't have a real OSS account to test. If you have a GCP account," - + "please change the configuration(BUCKET_NAME, OSS_ACCESS_KEY, OSS_SECRET_KEY, " - + "OSS_ENDPOINT) and enable this test.") +@EnabledIf(value = "ossIsConfigured", disabledReason = "OSS is not configured.") public class HadoopOSSCatalogIT extends HadoopCatalogIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopOSSCatalogIT.class); - public static final String BUCKET_NAME = "YOUR_BUCKET"; - public static final String OSS_ACCESS_KEY = "YOUR_OSS_ACCESS_KEY"; - public static final String OSS_SECRET_KEY = "YOUR_OSS_SECRET_KEY"; - public static final String OSS_ENDPOINT = "YOUR_OSS_ENDPOINT"; + public static final String BUCKET_NAME = System.getenv("OSS_BUCKET_NAME"); + public static final String OSS_ACCESS_KEY = System.getenv("OSS_ACCESS_KEY_ID"); + public static final String OSS_SECRET_KEY = System.getenv("OSS_SECRET_ACCESS_KEY"); + public static final String OSS_ENDPOINT = System.getenv("OSS_ENDPOINT"); @VisibleForTesting public void startIntegrationTest() throws Exception {} @@ -197,4 +195,11 @@ public void testCreateSchemaAndFilesetWithSpecialLocation() { // Delete catalog metalake.dropCatalog(localCatalogName, true); } + + protected static boolean ossIsConfigured() { + return StringUtils.isNotBlank(System.getenv("OSS_ACCESS_KEY_ID")) + && StringUtils.isNotBlank(System.getenv("OSS_SECRET_ACCESS_KEY")) + && StringUtils.isNotBlank(System.getenv("OSS_ENDPOINT")) + && StringUtils.isNotBlank(System.getenv("OSS_BUCKET_NAME")); + } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java index 808aafbaef5..e967a1c6af7 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java @@ -36,8 +36,6 @@ import java.util.Map; import java.util.Objects; import java.util.Properties; -import org.apache.commons.lang3.JavaVersion; -import org.apache.commons.lang3.SystemUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -65,6 +63,8 @@ import org.junit.jupiter.api.condition.EnabledIf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.shaded.org.apache.commons.lang3.JavaVersion; +import org.testcontainers.shaded.org.apache.commons.lang3.SystemUtils; @Tag("gravitino-docker-test") public class HadoopUserImpersonationIT extends BaseIT { diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index 95951459b03..15833aca01a 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -36,12 +36,21 @@ logger = logging.getLogger(__name__) -@unittest.skip("This test require GCS service account key file") +def oss_is_configured(): + return all( + [ + os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH") is not None, + os.environ.get("GCS_BUCKET_NAME") is not None, + ] + ) + + +@unittest.skipUnless(oss_is_configured(), "GCS is not configured.") class TestGvfsWithGCS(TestGvfsWithHDFS): # Before running this test, please set the make sure gcp-bundle-x.jar has been # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory - key_file = "your_key_file.json" - bucket_name = "your_bucket_name" + key_file = os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH") + bucket_name = os.environ.get("GCS_BUCKET_NAME") metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) def setUp(self): diff --git a/clients/client-python/tests/integration/test_gvfs_with_oss.py b/clients/client-python/tests/integration/test_gvfs_with_oss.py index 7b709b49580..97219a84146 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_oss.py +++ b/clients/client-python/tests/integration/test_gvfs_with_oss.py @@ -37,14 +37,25 @@ logger = logging.getLogger(__name__) -@unittest.skip("This test require oss service account") +def oss_is_configured(): + return all( + [ + os.environ.get("OSS_ACCESS_KEY_ID") is not None, + os.environ.get("OSS_SECRET_ACCESS_KEY") is not None, + os.environ.get("OSS_ENDPOINT") is not None, + os.environ.get("OSS_BUCKET_NAME") is not None, + ] + ) + + +@unittest.skipUnless(oss_is_configured(), "OSS is not configured.") class TestGvfsWithOSS(TestGvfsWithHDFS): # Before running this test, please set the make sure aliyun-bundle-x.jar has been # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory - oss_access_key = "your_access_key" - oss_secret_key = "your_secret_key" - oss_endpoint = "your_endpoint" - bucket_name = "your_bucket_name" + oss_access_key = os.environ.get("OSS_ACCESS_KEY_ID") + oss_secret_key = os.environ.get("OSS_SECRET_ACCESS_KEY") + oss_endpoint = os.environ.get("OSS_ENDPOINT") + bucket_name = os.environ.get("OSS_BUCKET_NAME") metalake_name: str = "TestGvfsWithOSS_metalake" + str(randint(1, 10000)) diff --git a/clients/client-python/tests/integration/test_gvfs_with_s3.py b/clients/client-python/tests/integration/test_gvfs_with_s3.py index ec059a88f37..fadcf352d46 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_s3.py +++ b/clients/client-python/tests/integration/test_gvfs_with_s3.py @@ -35,14 +35,25 @@ logger = logging.getLogger(__name__) -@unittest.skip("This test require S3 service account") +def s3_is_configured(): + return all( + [ + os.environ.get("S3_ACCESS_KEY_ID") is not None, + os.environ.get("S3_SECRET_ACCESS_KEY") is not None, + os.environ.get("S3_ENDPOINT") is not None, + os.environ.get("S3_BUCKET_NAME") is not None, + ] + ) + + +@unittest.skipUnless(s3_is_configured(), "S3 is not configured.") class TestGvfsWithS3(TestGvfsWithHDFS): # Before running this test, please set the make sure aws-bundle-x.jar has been # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory - s3_access_key = "your_access_key" - s3_secret_key = "your_secret_key" - s3_endpoint = "your_endpoint" - bucket_name = "your_bucket_name" + s3_access_key = os.environ.get("S3_ACCESS_KEY_ID") + s3_secret_key = os.environ.get("S3_SECRET_ACCESS_KEY") + s3_endpoint = os.environ.get("S3_ENDPOINT") + bucket_name = os.environ.get("S3_BUCKET_NAME") metalake_name: str = "TestGvfsWithS3_metalake" + str(randint(1, 10000)) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index e3567177007..b66fb34df37 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.gcs.fs.GCSFileSystemProvider; @@ -36,17 +37,16 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.condition.EnabledIf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Disabled( - "Disabled due to we don't have a real GCP account to test. If you have a GCP account," - + "please change the configuration(YOUR_KEY_FILE, YOUR_BUCKET) and enable this test.") +@EnabledIf(value = "isGCPConfigured", disabledReason = "GCP is not configured") public class GravitinoVirtualFileSystemGCSIT extends GravitinoVirtualFileSystemIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemGCSIT.class); - public static final String BUCKET_NAME = "YOUR_BUCKET"; - public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + public static final String BUCKET_NAME = System.getenv("GCS_BUCKET_NAME"); + public static final String SERVICE_ACCOUNT_FILE = System.getenv("GCS_SERVICE_ACCOUNT_JSON_PATH"); @BeforeAll public void startIntegrationTest() { @@ -141,4 +141,9 @@ protected String genStorageLocation(String fileset) { @Disabled( "GCS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} + + private static boolean isGCPConfigured() { + return StringUtils.isNotBlank(System.getenv("GCS_SERVICE_ACCOUNT_JSON_PATH")) + && StringUtils.isNotBlank(System.getenv("GCS_BUCKET_NAME")); + } } diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java index 257181dbfe7..e2218b5bc89 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java @@ -36,19 +36,19 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Disabled( - "Disabled due to we don't have a real OSS account to test. If you have a GCP account," - + "please change the configuration(BUCKET_NAME, OSS_ACCESS_KEY, OSS_SECRET_KEY, OSS_ENDPOINT) and enable this test.") +@EnabledIf(value = "ossIsConfigured", disabledReason = "OSS is not prepared") public class GravitinoVirtualFileSystemOSSIT extends GravitinoVirtualFileSystemIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemOSSIT.class); - public static final String BUCKET_NAME = "YOUR_BUCKET"; - public static final String OSS_ACCESS_KEY = "YOUR_OSS_ACCESS_KEY"; - public static final String OSS_SECRET_KEY = "YOUR_OSS_SECRET_KEY"; - public static final String OSS_ENDPOINT = "YOUR_OSS_ENDPOINT"; + public static final String BUCKET_NAME = System.getenv("OSS_BUCKET_NAME"); + public static final String OSS_ACCESS_KEY = System.getenv("OSS_ACCESS_KEY_ID"); + public static final String OSS_SECRET_KEY = System.getenv("OSS_SECRET_ACCESS_KEY"); + public static final String OSS_ENDPOINT = System.getenv("OSS_ENDPOINT"); @BeforeAll public void startIntegrationTest() { @@ -149,4 +149,11 @@ protected String genStorageLocation(String fileset) { @Disabled( "OSS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} + + protected static boolean ossIsConfigured() { + return StringUtils.isNotBlank(System.getenv("OSS_ACCESS_KEY_ID")) + && StringUtils.isNotBlank(System.getenv("OSS_SECRET_ACCESS_KEY")) + && StringUtils.isNotBlank(System.getenv("OSS_ENDPOINT")) + && StringUtils.isNotBlank(System.getenv("OSS_BUCKET_NAME")); + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6629c8897a3..79f953fa9a0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,10 +32,10 @@ airlift-json = "237" airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" -hadoop3 = "3.1.0" +hadoop3 = "3.3.0" hadoop3-gcs = "1.9.4-hadoop3" -hadoop3-aliyun = "3.1.0" -hadoop-minikdc = "3.3.6" +hadoop3-aliyun = "3.3.0" +hadoop-minikdc = "3.3.0" htrace-core4 = "4.1.0-incubating" httpclient5 = "5.2.1" mockserver = "5.15.0"