Skip to content

Commit

Permalink
end line
Browse files Browse the repository at this point in the history
  • Loading branch information
@CalvinKirs
CalvinKirs committed Sep 25, 2023
1 parent cdc8189 commit 6c06211
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 56 deletions.
43 changes: 14 additions & 29 deletions fe/fe-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ under the License.
<antlr4.version>4.9.3</antlr4.version>
<awssdk.version>2.17.257</awssdk.version>
<huaweiobs.version>3.1.1-hw-46</huaweiobs.version>
<tencentcos.version>3.3.5</tencentcos.version>
<tencentcos.version>3.3.6</tencentcos.version>
</properties>
<profiles>
<profile>
Expand Down Expand Up @@ -342,25 +342,9 @@ under the License.
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.client -->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.6</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<artifactId>netty-all</artifactId>
<groupId>io.netty</groupId>
</exclusion>
</exclusions>
<groupId>org.apache.doris</groupId>
<artifactId>doris-third-party</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.protocol -->
<dependency>
Expand Down Expand Up @@ -402,6 +386,7 @@ under the License.
<groupId>com.huaweicloud</groupId>
<artifactId>hadoop-huaweicloud</artifactId>
<version>${huaweiobs.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>
Expand All @@ -427,6 +412,10 @@ under the License.
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>spring-shade-5</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jetty</artifactId>
Expand Down Expand Up @@ -548,30 +537,35 @@ under the License.
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-core</artifactId>
<version>${paimon.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-common</artifactId>
<version>${paimon.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-format</artifactId>
<version>${paimon.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-s3</artifactId>
<version>${paimon.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-oss</artifactId>
<version>${paimon.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
Expand Down Expand Up @@ -628,16 +622,6 @@ under the License.
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-common</artifactId>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-hadoop-mr -->
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-hadoop-mr</artifactId>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<dependency>
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
Expand Down Expand Up @@ -682,6 +666,7 @@ under the License.
<artifactId>gcs-connector</artifactId>
<version>hadoop2-2.2.8</version>
<classifier>shaded</classifier>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.doris.datasource.property.constants.S3Properties;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.aliyun.oss.Constants;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.aws.s3.S3FileIO;
import org.apache.iceberg.catalog.TableIdentifier;
Expand All @@ -50,7 +49,8 @@ protected TableOperations newTableOps(TableIdentifier tableIdentifier) {

protected FileIO initializeFileIO(Map<String, String> properties, Configuration hadoopConf) {
// read from converted properties or default by old s3 aws properties
String endpoint = properties.getOrDefault(Constants.ENDPOINT_KEY, properties.get(S3Properties.Env.ENDPOINT));
String endpoint = properties.getOrDefault(OssProperties.getOrginalOssConfigKey(OssProperties.ENDPOINT),
properties.get(S3Properties.Env.ENDPOINT));
CloudCredential credential = new CloudCredential();
credential.setAccessKey(properties.getOrDefault(OssProperties.ACCESS_KEY,
properties.get(S3Properties.Env.ACCESS_KEY)));
Expand Down
26 changes: 12 additions & 14 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,6 @@
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
import org.apache.hadoop.fs.cosn.CosNConfigKeys;
import org.apache.hadoop.fs.cosn.CosNFileSystem;
import org.apache.hadoop.fs.obs.OBSConstants;
import org.apache.hadoop.fs.obs.OBSFileSystem;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider;
Expand Down Expand Up @@ -161,12 +156,15 @@ private static Map<String, String> convertToCompatibleS3Properties(Map<String, S
private static Map<String, String> convertToOBSProperties(Map<String, String> props,
CloudCredential credential) {
Map<String, String> obsProperties = Maps.newHashMap();
obsProperties.put(OBSConstants.ENDPOINT, props.get(ObsProperties.ENDPOINT));
obsProperties.put(ObsProperties.getOrinigalObsConfigKey(ObsProperties.ENDPOINT),
props.get(ObsProperties.ENDPOINT));
obsProperties.put(ObsProperties.FS.IMPL_DISABLE_CACHE, "true");
obsProperties.put("fs.obs.impl", getHadoopFSImplByScheme("obs"));
if (credential.isWhole()) {
obsProperties.put(OBSConstants.ACCESS_KEY, credential.getAccessKey());
obsProperties.put(OBSConstants.SECRET_KEY, credential.getSecretKey());
obsProperties.put(ObsProperties.getOrinigalObsConfigKey(ObsProperties.ACCESS_KEY),
credential.getAccessKey());
obsProperties.put(ObsProperties.getOrinigalObsConfigKey(ObsProperties.SECRET_KEY),
credential.getSecretKey());
}
if (credential.isTemporary()) {
obsProperties.put(ObsProperties.FS.SESSION_TOKEN, credential.getSessionToken());
Expand All @@ -181,11 +179,11 @@ private static Map<String, String> convertToOBSProperties(Map<String, String> pr

public static String getHadoopFSImplByScheme(String fsScheme) {
if (fsScheme.equalsIgnoreCase("obs")) {
return OBSFileSystem.class.getName();
return ObsProperties.ObsHdfsImpl;
} else if (fsScheme.equalsIgnoreCase("oss")) {
return AliyunOSSFileSystem.class.getName();
return OssProperties.OSS_HDFS_IMPL;
} else if (fsScheme.equalsIgnoreCase("cosn")) {
return CosNFileSystem.class.getName();
return CosProperties.CosHdfsImpl;
} else {
return S3AFileSystem.class.getName();
}
Expand Down Expand Up @@ -337,12 +335,12 @@ private static void rewriteHdfsOnOssProperties(Map<String, String> ossProperties

private static Map<String, String> convertToCOSProperties(Map<String, String> props, CloudCredential credential) {
Map<String, String> cosProperties = Maps.newHashMap();
cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY, props.get(CosProperties.ENDPOINT));
cosProperties.put(CosProperties.COSN_ENDPOINT_SUFFIX_KEY, props.get(CosProperties.ENDPOINT));
cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
if (credential.isWhole()) {
cosProperties.put(CosNConfigKeys.COSN_SECRET_ID_KEY, credential.getAccessKey());
cosProperties.put(CosNConfigKeys.COSN_SECRET_KEY_KEY, credential.getSecretKey());
cosProperties.put(CosProperties.COSN_SECRET_ID_KEY, credential.getAccessKey());
cosProperties.put(CosProperties.COSN_SECRET_KEY_KEY, credential.getSecretKey());
}
// session token is unsupported
for (Map.Entry<String, String> entry : props.entrySet()) {
Expand Down
6 changes: 6 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ public class CosProperties extends BaseProperties {
public static final String SECRET_KEY = "cos.secret_key";
public static final String REGION = "cos.region";
public static final String SESSION_TOKEN = "cos.session_token";
public static final String COSN_ENDPOINT_SUFFIX_KEY = "fs.cosn.bucket.endpoint_suffix";

public static final String COSN_SECRET_ID_KEY = "fs.cosn.userinfo.secretId";
public static final String COSN_SECRET_KEY_KEY = "fs.cosn.userinfo.secretKey";

public static final String CosHdfsImpl = "org.apache.hadoop.fs.cosn.CosNFileSystem";
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);


Expand Down
6 changes: 6 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,14 @@ public class ObsProperties extends BaseProperties {
public static final String ACCESS_KEY = "obs.access_key";
public static final String SECRET_KEY = "obs.secret_key";
public static final String SESSION_TOKEN = "obs.session_token";

public static final String ObsHdfsImpl = "org.apache.hadoop.fs.obs.OBSFileSystem";
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);

public static String getOrinigalObsConfigKey(String key) {
return OBS_PREFIX + key;
}

public static class FS {
public static final String SESSION_TOKEN = "fs.obs.session.token";
public static final String IMPL_DISABLE_CACHE = "fs.obs.impl.disable.cache";
Expand Down
7 changes: 7 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ public class OssProperties extends BaseProperties {
public static final String SECRET_KEY = "oss.secret_key";
public static final String SESSION_TOKEN = "oss.session_token";
public static final String OSS_HDFS_ENABLED = "oss.hdfs.enabled";

public static final String OSS_HDFS_IMPL = "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem";

public static String getOrginalOssConfigKey(String key) {
return OSS_PREFIX + key;
}

public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);

public static CloudCredential getCredential(Map<String, String> props) {
Expand Down
67 changes: 56 additions & 11 deletions fe/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ under the License.
<doris.home>${fe.dir}/../</doris.home>
<revision>1.2-SNAPSHOT</revision>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<doris.hive.catalog.shade.version>1.0.1</doris.hive.catalog.shade.version>
<doris.hive.catalog.shade.version>1.0.0-GA</doris.hive.catalog.shade.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<!--plugin parameters-->
Expand Down Expand Up @@ -513,8 +513,22 @@ under the License.
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>spring-shade-5</artifactId>
<version>${doris.hive.catalog.shade.version}</version>
</dependency>
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.166</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
Expand Down Expand Up @@ -680,6 +694,17 @@ under the License.
<groupId>org.apache.doris</groupId>
<artifactId>je</artifactId>
<version>${je.version}</version>
<exclusions>
<exclusion>
<groupId>org.checkerframework</groupId>
<artifactId>checker</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>doris-third-party</artifactId>
<version>${doris.hive.catalog.shade.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/de.jflex/jflex -->
<dependency>
Expand Down Expand Up @@ -1136,6 +1161,10 @@ under the License.
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -1196,6 +1225,14 @@ under the License.
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-protobuf_3_7</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -1271,8 +1308,26 @@ under the License.
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop2-compat</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-miscellaneous</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.9.13</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
Expand All @@ -1290,11 +1345,6 @@ under the License.
<artifactId>hudi-hadoop-mr</artifactId>
<version>${hudi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<version>${parquet.version}</version>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
Expand Down Expand Up @@ -1470,11 +1520,6 @@ under the License.
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
Expand Down

0 comments on commit 6c06211

Please sign in to comment.