Executor安全加固

Mu-L · Aug 18, 2020 · dcbab16 · dcbab16
1 parent 62ec97b
commit dcbab16
Show file tree

Hide file tree

Showing 31 changed files with 1,016 additions and 432 deletions.
diff --git a/az-core/src/main/java/azkaban/Constants.java b/az-core/src/main/java/azkaban/Constants.java
@@ -17,8 +17,6 @@
 package azkaban;
 
 import java.time.Duration;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Constants used in configuration files or shared among classes.
@@ -38,7 +36,6 @@
  */
 public class Constants {
 
-
   public static final String FLOW_PAUSED_MAX_TIME_MS = "flow.paused.max.time.ms";
 
   public static final long DEFAULT_FLOW_PAUSED_MAX_TIME = 1 * 60 * 60 * 1000;
@@ -163,6 +160,9 @@ public static class ConfigurationKeys {
 
     public static final String IS_METRICS_ENABLED = "azkaban.is.metrics.enabled";
 
+    public static final String IP_WHITELIST_ENABLED = "azkaban.ip.whiteList.enabled";
+    public static final String IP_WHITELIST = "azkaban.ip.whiteList";
+
     // User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users.
     // enduser -> myazkabanhost:443 -> proxy -> localhost:8081
     // when this parameters set then these parameters are used to generate email links.

diff --git a/az-webank-alerter/pom.xml b/az-webank-alerter/pom.xml
@@ -18,31 +18,11 @@
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-common</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
-          <groupId>org.slf4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-web-server</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>

diff --git a/az-webank-homepage/pom.xml b/az-webank-homepage/pom.xml
@@ -23,31 +23,11 @@
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-common</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
-          <groupId>org.slf4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-web-server</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>

diff --git a/az-webank-system-manager/conf/plugin.properties b/az-webank-system-manager/conf/plugin.properties
@@ -7,5 +7,8 @@ viewer.servlet.class=com.webank.wedatasphere.schedulis.system.servlet.SystemServ
 wtss.db.jdbc.url=[#SYSTEM_DB_URL]
 wtss.db.username=[#SYSTEM_DB_USERNAME]
 wtss.db.password=[#SYSTEM_DB_PASSWORD]
+esb.app.domain=[#ESB_APP_URL]
+esb.app.id=[#ESB_APP_ID]
+esb.app.token=[#ESB_APP_TOKEN]
 
 
diff --git a/az-webank-system-manager/pom.xml b/az-webank-system-manager/pom.xml
@@ -18,27 +18,11 @@
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-common</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-web-server</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>
@@ -99,15 +83,14 @@
     <dependency>
       <groupId>com.alibaba</groupId>
       <artifactId>easyexcel</artifactId>
-      <version>1.1.1</version>
+      <version>2.2.3</version>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
       <version>${guava.version}</version>
     </dependency>
 
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>

diff --git a/...rc/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java b/...rc/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java
@@ -97,7 +97,7 @@ public User getUser(String username, String password) throws UserManagerExceptio
         initUserAuthority(wtssUser, user);
 
       } catch (Exception e) {
-        logger.error("登录失败！异常信息：" + e);
+        logger.error("登录失败！异常信息：", e);
         throw new UserManagerException("Error User Name Or Password.");
       }
     }

diff --git a/az-webank-user-params/pom.xml b/az-webank-user-params/pom.xml
@@ -18,31 +18,11 @@
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-common</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
-          <groupId>org.slf4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>
       <artifactId>azkaban-web-server</artifactId>
       <version>${schedulis.version}</version>
-      <exclusions>
-        <exclusion>
-          <artifactId>fastjson</artifactId>
-          <groupId>com.alibaba</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.webank.wedatasphere.schedulis</groupId>

diff --git a/...src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java b/...src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java
@@ -122,7 +122,7 @@ public boolean checkWtssUserIsExist(String name){
         try {
             cout = jdbcExecutorLoader.findWtssUserByName(name);
         }catch (ExecutorManagerException e){
-            logger.error("can not found wtssuser by" + name + ", " + e);
+            logger.error("can not found wtssuser by" + name + ", ", e);
         }
         if(cout == 0){
             return false;

diff --git a/azkaban-common/pom.xml b/azkaban-common/pom.xml
@@ -20,10 +20,18 @@
       <version>${hadoop.version}</version>
       <exclusions>
         <exclusion>
-          <artifactId>netty</artifactId>
-          <groupId>io.netty</groupId>
+          <artifactId>jackson-databind</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-annotations</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-core</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
         </exclusion>
-	<exclusion>
+        <exclusion>
           <artifactId>slf4j-log4j12</artifactId>
           <groupId>org.slf4j</groupId>
         </exclusion>
@@ -99,22 +107,21 @@
           <artifactId>netty</artifactId>
           <groupId>io.netty</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>log4j</artifactId>
-          <groupId>log4j</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>slf4j-log4j12</artifactId>
-          <groupId>org.slf4j</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
       <exclusions>
-
+        <exclusion>
+          <artifactId>jackson-databind</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-annotations</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>log4j</artifactId>
           <groupId>log4j</groupId>
@@ -123,6 +130,10 @@
           <artifactId>slf4j-log4j12</artifactId>
           <groupId>org.slf4j</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>commons-beanutils</artifactId>
+          <groupId>commons-beanutils</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -131,10 +142,10 @@
       <version>${hadoop.version}</version>
       <exclusions>
         <exclusion>
-          <artifactId>netty</artifactId>
-          <groupId>io.netty</groupId>
+          <artifactId>jackson-databind</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
         </exclusion>
-	<exclusion>
+        <exclusion>
           <artifactId>log4j</artifactId>
           <groupId>log4j</groupId>
         </exclusion>
@@ -182,6 +193,12 @@
       <groupId>org.quartz-scheduler</groupId>
       <artifactId>quartz</artifactId>
       <version>${quartz.version}</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>c3p0</artifactId>
+          <groupId>c3p0</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -217,20 +234,6 @@
       <groupId>org.apache.velocity</groupId>
       <artifactId>velocity</artifactId>
       <version>${velocity.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.struts</groupId>
-          <artifactId>struts-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.struts</groupId>
-          <artifactId>struts-taglib</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.struts</groupId>
-          <artifactId>struts-tiles</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.velocity</groupId>
@@ -268,7 +271,17 @@
     <dependency>
       <groupId>com.alibaba</groupId>
       <artifactId>easyexcel</artifactId>
-      <version>1.1.1</version>
+      <version>2.2.3</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>poi</artifactId>
+          <groupId>org.apache.poi</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>poi-ooxml-schemas</artifactId>
+          <groupId>org.apache.poi</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>

diff --git a/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java b/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java
@@ -23,6 +23,7 @@ public interface ConnectorParams {
   public static final String EXECID_PARAM = "execid";
   public static final String SHAREDTOKEN_PARAM = "token";
   public static final String USER_PARAM = "user";
+  public static final String TOKEN_PARAM = "token";
 
   public static final String UPDATE_ACTION = "update";
   public static final String STATUS_ACTION = "status";

diff --git a/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java b/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java
@@ -16,8 +16,10 @@
 
 package azkaban.executor;
 
+import azkaban.Constants;
 import azkaban.utils.JSONUtils;
 import azkaban.utils.Pair;
+import azkaban.utils.Props;
 import com.google.inject.Inject;
 import java.io.IOException;
 import java.net.URI;
@@ -28,17 +30,20 @@
 import java.util.concurrent.TimeUnit;
 import javax.inject.Singleton;
 
+import com.webank.wedatasphere.schedulis.common.utils.JwtTokenUtils;
 import okhttp3.*;
 import org.codehaus.jackson.map.ObjectMapper;
 
 @Singleton
 public class ExecutorApiGateway {
 
   private final ExecutorApiClient apiClient;
+  private final Props azkProps;
 
   @Inject
-  public ExecutorApiGateway(final ExecutorApiClient apiClient) {
+  public ExecutorApiGateway(final ExecutorApiClient apiClient,final Props azkProps) {
     this.apiClient = apiClient;
+    this.azkProps = azkProps;
   }
 
   // FIXME change this method access as public type in order to outside package object can call this method.
@@ -82,6 +87,12 @@ public Map<String, Object> callWithExecutionId(final String host, final int port
           .valueOf(executionId)));
       paramList.add(new Pair<>(ConnectorParams.USER_PARAM, user));
 
+      if(this.azkProps.getBoolean(Constants.ConfigurationKeys.IP_WHITELIST_ENABLED,false)){
+        String dss_secret = azkProps.getString("dss.secret", "dws-wtss|WeBankBDPWTSS&DWS@2019");
+        String token = JwtTokenUtils.getToken(null,false,dss_secret,300);
+        paramList.add(new Pair<>(ConnectorParams.TOKEN_PARAM, token));
+      }
+
       return callForJsonObjectMap(host, port, "/executor", paramList);
     } catch (final IOException e) {
       throw new ExecutorManagerException(e);
@@ -131,6 +142,12 @@ private String callForJsonString(final String host, final int port, final String
     @SuppressWarnings("unchecked") final URI uri =
         ExecutorApiClient.buildUri(host, port, path, true);
 
+    if(this.azkProps.getBoolean(Constants.ConfigurationKeys.IP_WHITELIST_ENABLED,false)){
+      String dss_secret = azkProps.getString("dss.secret", "dws-wtss|WeBankBDPWTSS&DWS@2019");
+      String token = JwtTokenUtils.getToken(null,false,dss_secret,300);
+      paramList.add(new Pair<>(ConnectorParams.TOKEN_PARAM, token));
+    }
+
     return this.apiClient.httpPost(uri, paramList);
   }