diff --git a/az-core/src/main/java/azkaban/Constants.java b/az-core/src/main/java/azkaban/Constants.java index 188692b..ab43ea0 100644 --- a/az-core/src/main/java/azkaban/Constants.java +++ b/az-core/src/main/java/azkaban/Constants.java @@ -17,8 +17,6 @@ package azkaban; import java.time.Duration; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; /** * Constants used in configuration files or shared among classes. @@ -38,7 +36,6 @@ */ public class Constants { - public static final String FLOW_PAUSED_MAX_TIME_MS = "flow.paused.max.time.ms"; public static final long DEFAULT_FLOW_PAUSED_MAX_TIME = 1 * 60 * 60 * 1000; @@ -163,6 +160,9 @@ public static class ConfigurationKeys { public static final String IS_METRICS_ENABLED = "azkaban.is.metrics.enabled"; + public static final String IP_WHITELIST_ENABLED = "azkaban.ip.whiteList.enabled"; + public static final String IP_WHITELIST = "azkaban.ip.whiteList"; + // User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users. // enduser -> myazkabanhost:443 -> proxy -> localhost:8081 // when this parameters set then these parameters are used to generate email links. diff --git a/az-webank-alerter/pom.xml b/az-webank-alerter/pom.xml index c5ff9d9..3e440ea 100644 --- a/az-webank-alerter/pom.xml +++ b/az-webank-alerter/pom.xml @@ -18,31 +18,11 @@ com.webank.wedatasphere.schedulis azkaban-common ${schedulis.version} - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis azkaban-web-server ${schedulis.version} - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis diff --git a/az-webank-homepage/pom.xml b/az-webank-homepage/pom.xml index d0d1fdf..9309885 100755 --- a/az-webank-homepage/pom.xml +++ b/az-webank-homepage/pom.xml @@ -23,31 +23,11 @@ com.webank.wedatasphere.schedulis azkaban-common ${schedulis.version} - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis azkaban-web-server ${schedulis.version} - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis diff --git a/az-webank-system-manager/conf/plugin.properties b/az-webank-system-manager/conf/plugin.properties index 21b89c6..7045331 100755 --- a/az-webank-system-manager/conf/plugin.properties +++ b/az-webank-system-manager/conf/plugin.properties @@ -7,5 +7,8 @@ viewer.servlet.class=com.webank.wedatasphere.schedulis.system.servlet.SystemServ wtss.db.jdbc.url=[#SYSTEM_DB_URL] wtss.db.username=[#SYSTEM_DB_USERNAME] wtss.db.password=[#SYSTEM_DB_PASSWORD] +esb.app.domain=[#ESB_APP_URL] +esb.app.id=[#ESB_APP_ID] +esb.app.token=[#ESB_APP_TOKEN] diff --git a/az-webank-system-manager/pom.xml b/az-webank-system-manager/pom.xml index 6f152fb..0c88976 100755 --- a/az-webank-system-manager/pom.xml +++ b/az-webank-system-manager/pom.xml @@ -18,27 +18,11 @@ com.webank.wedatasphere.schedulis azkaban-common ${schedulis.version} - - - log4j - log4j - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis azkaban-web-server ${schedulis.version} - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis @@ -99,7 +83,7 @@ com.alibaba easyexcel - 1.1.1 + 2.2.3 com.google.guava @@ -107,7 +91,6 @@ ${guava.version} - org.apache.commons commons-lang3 diff --git a/az-webank-system-manager/src/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java b/az-webank-system-manager/src/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java index 817daba..064f885 100755 --- a/az-webank-system-manager/src/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java +++ b/az-webank-system-manager/src/main/java/com/webank/wedatasphere/schedulis/system/service/impl/SystemUserManager.java @@ -97,7 +97,7 @@ public User getUser(String username, String password) throws UserManagerExceptio initUserAuthority(wtssUser, user); } catch (Exception e) { - logger.error("登录失败！异常信息：" + e); + logger.error("登录失败！异常信息：", e); throw new UserManagerException("Error User Name Or Password."); } } diff --git a/az-webank-user-params/pom.xml b/az-webank-user-params/pom.xml index c4fc7b0..3185f97 100644 --- a/az-webank-user-params/pom.xml +++ b/az-webank-user-params/pom.xml @@ -18,31 +18,11 @@ com.webank.wedatasphere.schedulis azkaban-common ${schedulis.version} - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis azkaban-web-server ${schedulis.version} - - - fastjson - com.alibaba - - com.webank.wedatasphere.schedulis diff --git a/az-webank-user-params/src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java b/az-webank-user-params/src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java index 0e4835b..0c6ce23 100644 --- a/az-webank-user-params/src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java +++ b/az-webank-user-params/src/main/java/com/webank/wedatasphere/schedulis/userparams/service/UserParamsService.java @@ -122,7 +122,7 @@ public boolean checkWtssUserIsExist(String name){ try { cout = jdbcExecutorLoader.findWtssUserByName(name); }catch (ExecutorManagerException e){ - logger.error("can not found wtssuser by" + name + ", " + e); + logger.error("can not found wtssuser by" + name + ", ", e); } if(cout == 0){ return false; diff --git a/az-webank-user-params/src/web/js/userparams-manager.js b/az-webank-user-params/src/web/js/userparams-manager.js index 4b21e24..3f54ee9 100644 --- a/az-webank-user-params/src/web/js/userparams-manager.js +++ b/az-webank-user-params/src/web/js/userparams-manager.js @@ -657,6 +657,9 @@ azkaban.UpdateDepartmentGroupView = Backbone.View.extend({ }; var model = this.model; var successHandler = function (data) { + if (data.status && data.status == 'error') { + return false + } if (data.error) { $("#update-department-group-modal-error-msg").show(); $("#update-department-group-modal-error-msg").text(data.error); diff --git a/azkaban-common/pom.xml b/azkaban-common/pom.xml index 80e43a7..a70f15e 100755 --- a/azkaban-common/pom.xml +++ b/azkaban-common/pom.xml @@ -20,10 +20,18 @@ ${hadoop.version} - netty - io.netty + jackson-databind + com.fasterxml.jackson.core + + + jackson-annotations + com.fasterxml.jackson.core + + + jackson-core + com.fasterxml.jackson.core - + slf4j-log4j12 org.slf4j @@ -99,14 +107,6 @@ netty io.netty - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - @@ -114,7 +114,14 @@ hadoop-common ${hadoop.version} - + + jackson-databind + com.fasterxml.jackson.core + + + jackson-annotations + com.fasterxml.jackson.core + log4j log4j @@ -123,6 +130,10 @@ slf4j-log4j12 org.slf4j + + commons-beanutils + commons-beanutils + @@ -131,10 +142,10 @@ ${hadoop.version} - netty - io.netty + jackson-databind + com.fasterxml.jackson.core - + log4j log4j @@ -182,6 +193,12 @@ org.quartz-scheduler quartz ${quartz.version} + + + c3p0 + c3p0 + + @@ -217,20 +234,6 @@ org.apache.velocity velocity ${velocity.version} - - - org.apache.struts - struts-core - - - org.apache.struts - struts-taglib - - - org.apache.struts - struts-tiles - - org.apache.velocity @@ -268,7 +271,17 @@ com.alibaba easyexcel - 1.1.1 + 2.2.3 + + + poi + org.apache.poi + + + poi-ooxml-schemas + org.apache.poi + + diff --git a/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java b/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java index 5cc357c..e68834d 100755 --- a/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java +++ b/azkaban-common/src/main/java/azkaban/executor/ConnectorParams.java @@ -23,6 +23,7 @@ public interface ConnectorParams { public static final String EXECID_PARAM = "execid"; public static final String SHAREDTOKEN_PARAM = "token"; public static final String USER_PARAM = "user"; + public static final String TOKEN_PARAM = "token"; public static final String UPDATE_ACTION = "update"; public static final String STATUS_ACTION = "status"; diff --git a/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java b/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java index fa0c091..b7fd8d4 100755 --- a/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java +++ b/azkaban-common/src/main/java/azkaban/executor/ExecutorApiGateway.java @@ -16,8 +16,10 @@ package azkaban.executor; +import azkaban.Constants; import azkaban.utils.JSONUtils; import azkaban.utils.Pair; +import azkaban.utils.Props; import com.google.inject.Inject; import java.io.IOException; import java.net.URI; @@ -28,6 +30,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Singleton; +import com.webank.wedatasphere.schedulis.common.utils.JwtTokenUtils; import okhttp3.*; import org.codehaus.jackson.map.ObjectMapper; @@ -35,10 +38,12 @@ public class ExecutorApiGateway { private final ExecutorApiClient apiClient; + private final Props azkProps; @Inject - public ExecutorApiGateway(final ExecutorApiClient apiClient) { + public ExecutorApiGateway(final ExecutorApiClient apiClient,final Props azkProps) { this.apiClient = apiClient; + this.azkProps = azkProps; } // FIXME change this method access as public type in order to outside package object can call this method. @@ -82,6 +87,12 @@ public Map callWithExecutionId(final String host, final int port .valueOf(executionId))); paramList.add(new Pair<>(ConnectorParams.USER_PARAM, user)); + if(this.azkProps.getBoolean(Constants.ConfigurationKeys.IP_WHITELIST_ENABLED,false)){ + String dss_secret = azkProps.getString("dss.secret", "dws-wtss|WeBankBDPWTSS&DWS@2019"); + String token = JwtTokenUtils.getToken(null,false,dss_secret,300); + paramList.add(new Pair<>(ConnectorParams.TOKEN_PARAM, token)); + } + return callForJsonObjectMap(host, port, "/executor", paramList); } catch (final IOException e) { throw new ExecutorManagerException(e); @@ -131,6 +142,12 @@ private String callForJsonString(final String host, final int port, final String @SuppressWarnings("unchecked") final URI uri = ExecutorApiClient.buildUri(host, port, path, true); + if(this.azkProps.getBoolean(Constants.ConfigurationKeys.IP_WHITELIST_ENABLED,false)){ + String dss_secret = azkProps.getString("dss.secret", "dws-wtss|WeBankBDPWTSS&DWS@2019"); + String token = JwtTokenUtils.getToken(null,false,dss_secret,300); + paramList.add(new Pair<>(ConnectorParams.TOKEN_PARAM, token)); + } + return this.apiClient.httpPost(uri, paramList); } diff --git a/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java b/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java index 9ff62ae..a27bb3e 100755 --- a/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java +++ b/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java @@ -23,6 +23,7 @@ import com.webank.wedatasphere.schedulis.common.jobExecutor.utils.SystemBuiltInParamJodeTimeUtils; import com.webank.wedatasphere.schedulis.common.log.LogFilterEntity; +import com.webank.wedatasphere.schedulis.common.utils.JwtTokenUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.LoggerFactory; import org.slf4j.Logger; @@ -1086,6 +1087,17 @@ public void setFlowFailed(ExecutableFlow exFlow, String userId, List claimMaps, String encryKey, int secondTimeOut) { + return getToken(claimMaps, true, encryKey, secondTimeOut); + } + + /** + * 获取token + * + * @param claimMaps + * @param isJsonMpas + * @param encryKey + * @param secondTimeOut + * @return + */ + public static String getToken(Map claimMaps, boolean isJsonMpas, String encryKey, int secondTimeOut) { + + if (isJsonMpas) { + claimMaps.forEach((key, val) -> { + claimMaps.put(key, GsonUtils.toJson(val)); + }); + } + long currentTime = System.currentTimeMillis(); + byte[] apiKeySecretBytes = encryKey.getBytes(Charsets.UTF_8); + return Jwts.builder() + .setId(UUID.randomUUID().toString()) + .setIssuedAt(new Date(currentTime)) //签发时间 + .setExpiration(new Date(currentTime + secondTimeOut * 1000)) //过期时间戳 + .setSubject("webserver_to_executorserver") //说明 + .setAudience("executorservercontainer") //接收用户 + .setIssuer("webservercontainer") //签发者信息 + .compressWith(CompressionCodecs.GZIP) //数据压缩方式 + .signWith(SignatureAlgorithm.HS256, apiKeySecretBytes) //加密方式 + .addClaims(claimMaps) //cla信息 + .compact(); + } + + /** + * 获取token中的claims信息 + * + * @param token + * @param encryKey + * @return + */ + private static Jws getJws(String token, String encryKey) { + byte[] apiKeySecretBytes = encryKey.getBytes(Charsets.UTF_8); + return Jwts.parser() + .setSigningKey(apiKeySecretBytes) + .parseClaimsJws(token); + } + + /** + * 获取token中签名信息 + * + * @param token + * @param encryKey + * @return + */ + public static String getSignature(String token, String encryKey) { + try { + return getJws(token, encryKey).getSignature(); + } catch (Exception ex) { + return ""; + } + } + + /** + * 获取token中head信息 + * + * @param token + * @param encryKey + * @return + */ + public static JwsHeader getHeader(String token, String encryKey) { + try { + return getJws(token, encryKey).getHeader(); + } catch (Exception ex) { + return null; + } + } + + /** + * 获取payload body信息 + * + * @param token + * @param encryKey + * @return + */ + public static Claims getClaimsBody(String token, String encryKey) { + return getJws(token, encryKey).getBody(); + } + + /** + * 获取body某个值 + * + * @param token + * @param encryKey + * @param key + * @return + */ + public static Object getVal(String token, String encryKey, String key) { + return getJws(token, encryKey).getBody().get(key); + } + + /** + * 获取body某个值，json字符转实体 + * + * @param token + * @param encryKey + * @param key + * @param tClass + * @param + * @return + */ + public static T getValByT(String token, String encryKey, String key, Class tClass) { + try { + String strJson = getVal(token, encryKey, key).toString(); + return GsonUtils.fromJson(strJson, tClass); + } catch (Exception ex) { + return null; + } + } + + /** + * 是否过期 + * + * @param token + * @param encryKey + * @return + */ + public static boolean isExpiration(String token, String encryKey) { + try { + return getClaimsBody(token, encryKey) + .getExpiration() + .before(new Date()); + } catch (ExpiredJwtException ex) { + return true; + } + } + + /** + * 获取说明信息 + * + * @param token + * @param encryKey + * @param key + * @return + */ + public static String getSubject(String token, String encryKey) { + try { + return getClaimsBody(token, encryKey).getSubject(); + } catch (Exception ex) { + return ""; + } + } + + public static void main(String[] args) { + String token = getToken(null,false,"dws-wtss|WeBankBDPWTSS&DWS@2019",1 * 7 * 24 * 60 * 60 ); + System.out.println("testEncode: " + token); + + Claims claims = getJws(token,"dws-wtss|WeBankBDPWTSS&DWS@2019").getBody(); + System.out.println("ID: " + claims.getId()); + System.out.println("Subject: " + claims.getSubject()); + System.out.println("Issuer: " + claims.getIssuer()); + System.out.println("Audience: " + claims.getAudience()); + System.out.println("Expiration: " + claims.getExpiration()); + + } +} \ No newline at end of file diff --git a/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/RSAUtils.java b/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/RSAUtils.java new file mode 100644 index 0000000..8e7f7bf --- /dev/null +++ b/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/RSAUtils.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.schedulis.common.utils; + +import javax.crypto.Cipher; +import java.security.*; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; + + +/** + * @author georgeqiao + */ +public class RSAUtils { + /** + * 密钥长度 于原文长度对应 以及越长速度越慢 + */ + private final static int KEY_SIZE = 1024; + /** + * 公钥与私钥 + */ + private static String publicKeyString = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCo14viXWDm19hwsCpSGiaehrF+oBbGq7tIsgfepjkI0iC8HrZ/vo/YsHw+G3yGgEC2oBsHnqUDs/f2k+1FK6EIm5nBnosXpSTtCVDfP8wkwBuTWPdmnAxNQyv7aH2e7BLdfEctYLAdw8atahM2WV9rFZBQzCyheeJ3nI4LCBSzMwIDAQAB"; + private static String privateKeyString = "MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAKjXi+JdYObX2HCwKlIaJp6GsX6gFsaru0iyB96mOQjSILwetn++j9iwfD4bfIaAQLagGweepQOz9/aT7UUroQibmcGeixelJO0JUN8/zCTAG5NY92acDE1DK/tofZ7sEt18Ry1gsB3Dxq1qEzZZX2sVkFDMLKF54necjgsIFLMzAgMBAAECgYEAiQG/ZQRY6XklDOwmq1DFHcY2qYXGdZhM9QRiFm3TwjCgl4ZkmOxNVYyAhPVQ6uOPn6HzzQ8S4BpdkB0hYAuzMzMHnyhvv/MsH8V3WbJx0kC4If3UsSa0eANizrZaM3CwJ8ehvN2T73Z0B2iTp8Ocl3Wr74ULKC1y5pJnLC6dIMECQQDbhMVqIwPU+VftMEwIik0BDXHSFL4Vjh2rD9AVkvyWKV0qehYHGsXdlG4XlNUc0zAw8oQVFk9t6XGbMEWnk8QJAkEAxObH8gTy7iGYEUprxbEAJq0SJtOk3qfIngICpqdY5qgWqpl8BS7rSZhKswGLsrdY5rfYr/Mngj5cpe72lcbkWwJBAM+RRt4qR8hNEV/9CBgXNeLl5JdB988H93O12wtbVi1i5W5xzHxhS3FOlZ8Eo1LDOtE9r7kExIxobXzRczuWlIkCQQCJIBO1N78bGig2OnbtuYPaa4ONqK1UJtMvP0UrXLYsBHmsm7FkRrWzjizPl077ynZOT1DH0HX+XYHWSaJO0rGrAkBxQjicw5YAxaz0+Vl/HIFSCT4R4BBKDgSmwHf1yOUOIhspA2AOFdCByrmNg2gQ+Pn0ADHN6N3ke42eAt7GiI4T"; + + public static void genKeyPair() throws NoSuchAlgorithmException { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(KEY_SIZE, new SecureRandom()); + KeyPair keyPair = keyPairGen.generateKeyPair(); + RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate(); + RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic(); + publicKeyString = Base64.getEncoder().encodeToString(publicKey.getEncoded()); + privateKeyString = Base64.getEncoder().encodeToString(privateKey.getEncoded()); + + } + + public static String encrypt(String str, String publicKey) throws Exception { + byte[] decoded = Base64.getDecoder().decode(publicKey); + RSAPublicKey pubKey = (RSAPublicKey) KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(decoded)); + Cipher cipher = Cipher.getInstance("RSA"); + cipher.init(Cipher.ENCRYPT_MODE, pubKey); + String outStr = Base64.getEncoder().encodeToString(cipher.doFinal(str.getBytes("UTF-8"))); + return outStr; + } + + public static String decrypt(String str, String privateKey) throws Exception { + byte[] inputByte = Base64.getDecoder().decode(str); + byte[] decoded = Base64.getDecoder().decode(privateKey); + RSAPrivateKey priKey = (RSAPrivateKey) KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(decoded)); + Cipher cipher = Cipher.getInstance("RSA"); + cipher.init(Cipher.DECRYPT_MODE, priKey); + String outStr = new String(cipher.doFinal(inputByte)); + return outStr; + } + public static void main(String[] args) throws Exception { +// genKeyPair(); +// String message = "Bdp_wtss@We_bank@ABCD~!@#$"; +// String message = args[0]; +// System.out.println("原文:" + message) +// String messageEn = encrypt(message, publicKeyString); +// System.out.println("密文:" + messageEn); + String str = "DIjK7zXeIpNEDrt7f8eh9bnaP7KqcYgEEDGmxuXd9YnXPh7rkR9cTPfASyZg75tkvRS1YakFvWB5rC4qqHslUzYcQX/mORcB1UGbqPLcNC7uLCGD5zLOarcjO6 3e94Z/lIsipLRGwior1UiZdDIoFly3SXMINjbHwLWd0IMV64="; + + str = str.replaceAll(" ","+"); + + System.out.println("str:" + str); + String messageDe = decrypt(str, privateKeyString); + System.out.println("解密:" + messageDe); + } +} \ No newline at end of file diff --git a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/az-webank-system-manager-zh_CN.json b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/az-webank-system-manager-zh_CN.json index 840650b..274b040 100755 --- a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/az-webank-system-manager-zh_CN.json +++ b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/az-webank-system-manager-zh_CN.json @@ -24,7 +24,7 @@ "invalidAddRealNameUserProxy": "新增用户失败,实名用户仅可添加自己作为代理,请将代理用户选项置为空,或者将代理用户设置为:", "invalidUpdateRealNameUserProxy": "更新用户失败,实名用户仅可添加自己作为代理,请将代理用户选项置为空,或者将代理用户设置为:", "invalidUpdateSystemUserProxy": "更新用户失败,系统用户仅可添加自己作为代理,请将代理用户选项置为空,或者将代理用户设置为:", - "invalidName": "]输入不合法,合法用户名示例如下(大小写敏感):运维账号:WTSS_纯英文部门代码_编号(WTSS_DBDP_01); 普通系统账号:hduser编号(hduser01).", + "invalidName": "]输入不合法,合法用户名示例如下(大小写敏感):运维账号:WTSS_纯英文部门代码_编号(WTSS_DBDP_01); 普通系统账号:hduser编号(hduser01)", "existDep": "该部门已存在.", "noParentDep": "父部门不存在.", "plsInputDepId": "请输入部门ID.", diff --git a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-en_US.json b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-en_US.json index 400edde..f1a149f 100755 --- a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-en_US.json +++ b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-en_US.json @@ -734,7 +734,8 @@ "haveHisReRun": "History Re-Run job have been existed.", "submitHisReRunSuccess": " submit history Re-Run job success.", "submitHisReRunFail": " submit history Re-Run job failed.", - "resolveSlaFailed": "Resolve Sla alert settings failed." + "resolveSlaFailed": "Resolve Sla alert settings failed.", + "permissionForAction": "You have no Execute or Schedule permission for project: " }, "azkaban.webapp.servlet.ProjectManagerServlet": { "project": "Project[", @@ -744,6 +745,7 @@ "flow": "Flow[", "job": "Job[", "config": "Configuration[", + "permissionForAction": "You have no Execute or Schedule permission for project: ", "uploadJobCoverFieldError": "Upload job cover field error", "getLastRunFailed": "Get the last flow execution Failed", "noPerAccessProject": "No permission access to this project: ", diff --git a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-zh_CN.json b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-zh_CN.json index 525a642..8b5d1c9 100755 --- a/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-zh_CN.json +++ b/azkaban-common/src/main/resources/com.webank.wedatasphere.schedulis.i18n.conf/azkaban-web-server-zh_CN.json @@ -734,7 +734,8 @@ "haveHisReRun": "历史重跑任务已存在.", "submitHisReRunSuccess": " 提交历史重跑任务成功.", "submitHisReRunFail": " 提交历史重跑任务失败.", - "resolveSlaFailed": "解析sla告警设置失败." + "resolveSlaFailed": "解析sla告警设置失败.", + "permissionForAction": "你没有权限执行或者调度这个项目: " }, "azkaban.webapp.servlet.ProjectManagerServlet": { "project": "项目 ", @@ -744,6 +745,7 @@ "flow": " 工作流 ", "job": "任务 ", "config": "配置 ", + "permissionForAction": "你没有权限执行或者调度这个项目: ", "uploadJobCoverFieldError": "上传任务覆盖属性错误.", "getLastRunFailed": "获取项目最后一次执行工作流失败.", "noPerAccessProject": "没有权限查看项目 ", diff --git a/azkaban-exec-server/pom.xml b/azkaban-exec-server/pom.xml index 825a546..d540d2e 100644 --- a/azkaban-exec-server/pom.xml +++ b/azkaban-exec-server/pom.xml @@ -19,6 +19,12 @@ com.webank.wedatasphere.schedulis az-core ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis @@ -26,16 +32,8 @@ ${schedulis.version} - netty - io.netty - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j + * + * @@ -44,12 +42,24 @@ com.webank.wedatasphere.schedulis azkaban-spi ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis azkaban-db ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis @@ -57,44 +67,8 @@ ${schedulis.version} - jetty-all - org.eclipse.jetty.aggregate - - - hive-common - org.apache.hive - - - log4j-api - org.apache.logging.log4j - - - log4j-1.2-api - org.apache.logging.log4j - - - log4j-core - org.apache.logging.log4j - - - log4j-slf4j-impl - org.apache.logging.log4j - - - log4j-web - org.apache.logging.log4j - - - netty - io.netty - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j + * + * @@ -130,21 +104,6 @@ jetty-servlet ${jetty.version} - - org.eclipse.jetty - jetty-http - ${jetty.version} - - - org.eclipse.jetty - jetty-io - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - joda-time @@ -233,30 +192,31 @@ ${jsr305.version} - org.apache.kafka - kafka-log4j-appender - ${kafkaLog4jAppender.version} + org.quartz-scheduler + quartz + ${quartz.version} - log4j - log4j + c3p0 + c3p0 - slf4j-log4j12 + slf4j-api org.slf4j - - org.quartz-scheduler - quartz - ${quartz.version} - io.dropwizard.metrics metrics-core ${dropwizard.core.version} + + + slf4j-api + org.slf4j + + @@ -268,6 +228,12 @@ io.dropwizard.metrics metrics-jvm ${dropwizard.core.version} + + + slf4j-api + org.slf4j + + @@ -289,20 +255,6 @@ org.apache.velocity velocity ${velocity.version} - - - org.apache.struts - struts-core - - - org.apache.struts - struts-taglib - - - org.apache.struts - struts-tiles - - org.slf4j @@ -317,7 +269,6 @@ test - com.squareup.okhttp3 okhttp @@ -327,7 +278,53 @@ com.webank.wedatasphere.dss dss-linkis-node-execution - 0.7.0 + ${dss.version} + + + jackson-annotations + com.fasterxml.jackson.core + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + log4j + log4j + + + jetty-servlet + org.eclipse.jetty + + + ooxml-schemas + org.apache.poi + + + poi + org.apache.poi + + + poi-ooxml-schemas + org.apache.poi + + + poi-ooxml + org.apache.poi + + + gson + com.google.code.gson + + + slf4j-api + org.slf4j + + diff --git a/azkaban-exec-server/src/conf/azkaban.properties b/azkaban-exec-server/src/conf/azkaban.properties index 4f7c13b..3f66b0f 100644 --- a/azkaban-exec-server/src/conf/azkaban.properties +++ b/azkaban-exec-server/src/conf/azkaban.properties @@ -21,6 +21,7 @@ executor.maxThreads=60 executor.port=12321 executor.flow.threads=30 jetty.headerBufferSize=65536 +jetty.send.server.version=false flow.num.job.threads=50 checkers.num.threads=10 executor.server.id=[#SERVER_ID] @@ -63,3 +64,6 @@ proxy.user.lock.down=true # Automatically resume execution interval after pause. flow.paused.max.time.ms=3600000 +azkaban.ip.whiteList.enabled=false +azkaban.ip.whiteList= +dss.secret= diff --git a/azkaban-exec-server/src/main/bash/internal/internal-start-executor.sh b/azkaban-exec-server/src/main/bash/internal/internal-start-executor.sh index b6e318e..14a441f 100644 --- a/azkaban-exec-server/src/main/bash/internal/internal-start-executor.sh +++ b/azkaban-exec-server/src/main/bash/internal/internal-start-executor.sh @@ -1,6 +1,5 @@ #!/bin/bash -export JAVA_HOME="/nemo/jdk8" verify_java_home() { if [ -z "$JAVA_HOME" ]; then cat 1>&2 <> {} , path >> {}",servletRequest.getRemoteAddr(),((HttpServletRequest) servletRequest).getRequestURI()); + return; + } + }else{ + logger.error("Illegal access without token detected, ip >> {} , path >> {}",servletRequest.getRemoteAddr(),((HttpServletRequest) servletRequest).getRequestURI()); + return; + } + } catch (Exception e) { + logger.error("a fatal error had happen when execute ExecutorAccessFilter, caused by:", e); + } + } + + @Override + public void destroy() { + } +} \ No newline at end of file diff --git a/azkaban-hadoop-security-plugin/pom.xml b/azkaban-hadoop-security-plugin/pom.xml index 1da194a..3e090e4 100644 --- a/azkaban-hadoop-security-plugin/pom.xml +++ b/azkaban-hadoop-security-plugin/pom.xml @@ -24,20 +24,6 @@ com.webank.wedatasphere.schedulis azkaban-common ${schedulis.version} - - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - - - fastjson - com.alibaba - - @@ -49,14 +35,6 @@ netty io.netty - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - @@ -68,10 +46,6 @@ netty io.netty - - slf4j-log4j12 - org.slf4j - @@ -83,14 +57,6 @@ netty io.netty - - log4j - log4j - - - slf4j-log4j12 - org.slf4j - @@ -106,22 +72,6 @@ eigenbase eigenbase-properties - - log4j - log4j - - - log4j-slf4j-impl - org.apache.logging.log4j - - - log4j-api - org.apache.logging.log4j - - - log4j-core - org.apache.logging.log4j - diff --git a/azkaban-jobtype/pom.xml b/azkaban-jobtype/pom.xml index 02293e4..62ecf92 100644 --- a/azkaban-jobtype/pom.xml +++ b/azkaban-jobtype/pom.xml @@ -32,14 +32,6 @@ netty io.netty - - log4j - log4j - - - fastjson - com.alibaba - @@ -139,10 +131,6 @@ netty io.netty - - log4j - log4j - @@ -173,10 +161,6 @@ netty io.netty - - log4j - log4j - @@ -214,12 +198,6 @@ org.apache.hive hive-metastore ${hive.version} - - - log4j - log4j - - org.apache.hive @@ -261,10 +239,6 @@ log4j-slf4j-impl org.apache.logging.log4j - - log4j - log4j - @@ -280,21 +254,6 @@ netty io.netty - - log4j - log4j - - - - - org.apache.kafka - kafka-log4j-appender - ${kafkaLog4jAppender.version} - - - slf4j-log4j12 - org.slf4j - diff --git a/azkaban-web-server/pom.xml b/azkaban-web-server/pom.xml index b79e4e6..f7cb822 100644 --- a/azkaban-web-server/pom.xml +++ b/azkaban-web-server/pom.xml @@ -20,6 +20,12 @@ com.webank.wedatasphere.schedulis az-core ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis @@ -27,8 +33,8 @@ ${schedulis.version} - fastjson - com.alibaba + * + * @@ -36,16 +42,34 @@ com.webank.wedatasphere.schedulis azkaban-db ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis azkaban-spi ${schedulis.version} + + + * + * + + com.webank.wedatasphere.schedulis az-flow-trigger-dependency-plugin ${schedulis.version} + + + * + * + + @@ -62,13 +86,42 @@ com.linkedin.pegasus restli-server ${restli.version} + + + jackson-core + com.fasterxml.jackson.core + + + netty + org.jboss.netty + + + commons-compress + org.apache.commons + + + jackson-core-asl + org.codehaus.jackson + + + jackson-mapper-asl + org.codehaus.jackson + + com.linkedin.pegasus restli-tools ${restli.version} - + + com.fasterxml.jackson.core + jackson-databind + + + jackson-core + com.fasterxml.jackson.core + log4j log4j @@ -93,7 +146,42 @@ org.quartz-scheduler quartz ${quartz.version} + + + c3p0 + c3p0 + + + + + + com.mchange + c3p0 + ${c3p0.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml-schemas + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + + joda-time joda-time @@ -139,21 +227,7 @@ jetty-servlet ${jetty.version} - - org.eclipse.jetty - jetty-http - ${jetty.version} - - - org.eclipse.jetty - jetty-io - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - + commons-collections @@ -164,6 +238,20 @@ org.apache.commons commons-dbcp2 ${commons.dbcp2.version} + + + org.apache.struts + struts-core + + + org.apache.struts + struts-taglib + + + org.apache.struts + struts-tiles + + commons-dbutils @@ -196,20 +284,6 @@ org.apache.velocity velocity ${velocity.version} - - - org.apache.struts - struts-core - - - org.apache.struts - struts-taglib - - - org.apache.struts - struts-tiles - - org.apache.velocity @@ -228,6 +302,10 @@ org.apache.struts struts-tiles + + commons-beanutils + commons-beanutils + @@ -245,6 +323,7 @@ grizzly-framework 2.3.32 + org.forgerock.commons i18n-slf4j @@ -287,12 +366,64 @@ com.alibaba easyexcel - 1.1.1 + 2.2.3 + + + poi + org.apache.poi + + + poi-ooxml-schemas + org.apache.poi + + com.webank.wedatasphere.dss dss-appjoint-auth - 0.6.0 + ${dss.version} + + + jackson-annotations + com.fasterxml.jackson.core + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + log4j + log4j + + + jetty-servlet + org.eclipse.jetty + + + commons-beanutils + commons-beanutils + + + ooxml-schemas + org.apache.poi + + + poi + org.apache.poi + + + poi-ooxml-schemas + org.apache.poi + + + poi-ooxml + org.apache.poi + + diff --git a/azkaban-web-server/src/conf/azkaban.properties b/azkaban-web-server/src/conf/azkaban.properties index 7ab7398..52bad7a 100644 --- a/azkaban-web-server/src/conf/azkaban.properties +++ b/azkaban-web-server/src/conf/azkaban.properties @@ -31,6 +31,7 @@ jetty.use.ssl=false jetty.maxThreads=25 jetty.ssl.port=8443 jetty.port=8080 +jetty.send.server.version=false jetty.keystore=keystore/azkabanjetty.keystore jetty.password=hadoop jetty.keypassword=hadoop @@ -65,6 +66,8 @@ lockdown.create.projects=false wtss.project.privilege.check=false +department.maintainer.check.switch=true + cache.directory=cache # JMX stats @@ -82,6 +85,10 @@ ladp.port=2389 azkaban.header.referer.url=https:// nginx.ssl.module=close +dss.secret= +wtss.private.key= +azkaban.ip.whiteList.enabled=false +azkaban.ip.whiteList= #HA settings webserver.ha.model=false diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/AzkabanWebServer.java b/azkaban-web-server/src/main/java/azkaban/webapp/AzkabanWebServer.java index f5c91e8..3669708 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/AzkabanWebServer.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/AzkabanWebServer.java @@ -16,6 +16,52 @@ package azkaban.webapp; +import azkaban.executor.Status; +import azkaban.server.HttpRequestUtils; +import com.google.inject.Guice; +import com.google.inject.Injector; + +import com.linkedin.restli.server.RestliServlet; + +import joptsimple.internal.Strings; +import org.apache.commons.lang.StringUtils; +import org.eclipse.jetty.io.EndPoint; +import org.eclipse.jetty.server.HttpChannel; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.IPAccessHandler; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import org.apache.velocity.app.VelocityEngine; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.joda.time.DateTimeZone; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Constructor; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.management.MBeanInfo; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import javax.servlet.DispatcherType; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import static azkaban.ServiceProvider.SERVICE_PROVIDER; import static java.util.Objects.requireNonNull; @@ -30,7 +76,6 @@ import azkaban.executor.ExecutorManager; import azkaban.executor.ExecutorManagerAdapter; import azkaban.executor.ExecutorManagerException; -import azkaban.executor.Status; import azkaban.flowtrigger.FlowTriggerService; import azkaban.flowtrigger.quartz.FlowTriggerScheduler; import azkaban.jmx.JmxExecutionController; @@ -73,9 +118,6 @@ import azkaban.webapp.servlet.StatsServlet; import azkaban.webapp.servlet.StatusServlet; import azkaban.webapp.servlet.TriggerManagerServlet; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.linkedin.restli.server.RestliServlet; import com.webank.wedatasphere.schedulis.common.executor.ExecutionCycle; import com.webank.wedatasphere.schedulis.common.executor.ExecutorManagerHA; import com.webank.wedatasphere.schedulis.common.jmx.JmxExecutorManagerAdapter; @@ -83,17 +125,6 @@ import com.webank.wedatasphere.schedulis.web.webapp.LocaleFilter; import com.webank.wedatasphere.schedulis.web.webapp.servlet.CycleServlet; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.management.ManagementFactory; -import java.lang.reflect.Constructor; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -101,24 +132,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.concurrent.CompletableFuture; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.management.MBeanInfo; -import javax.management.MBeanServer; -import javax.management.ObjectName; -import javax.servlet.DispatcherType; -import joptsimple.internal.Strings; -import org.apache.commons.lang.StringUtils; -import org.apache.velocity.app.VelocityEngine; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.joda.time.DateTimeZone; -import org.slf4j.LoggerFactory; -import org.slf4j.Logger; /** * The Azkaban Jetty server class @@ -330,8 +344,7 @@ private static void loadViewerPlugins(final ServletContextHandler root, final St final ArrayList jarPaths = new ArrayList<>(); for (final File pluginDir : pluginDirs) { if (!pluginDir.exists()) { - logger.error("Error viewer plugin path " + pluginDir.getPath() - + " doesn't exist."); + logger.error("Error viewer plugin path " + pluginDir.getPath() + " doesn't exist."); continue; } @@ -801,6 +814,41 @@ private void configureRoutes() throws TriggerManagerException { getTriggerManager().start(); root.setAttribute(Constants.AZKABAN_SERVLET_CONTEXT_KEY, this); + + try { + if(this.props.getBoolean(ConfigurationKeys.IP_WHITELIST_ENABLED,false)){ + String whiteListStr = this.props.getString(ConfigurationKeys.IP_WHITELIST, ""); + String[] whiteListArr = whiteListStr.split(","); + IPAccessHandler ipAccessHandler = new IPAccessHandler(){ + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + HttpChannel channel = baseRequest.getHttpChannel(); + if (channel != null) { + EndPoint endp = channel.getEndPoint(); + if (endp != null) { + InetSocketAddress address = endp.getRemoteAddress(); + if("/executor".equals(baseRequest.getMetaData().getURI().getDecodedPath()) + && HttpRequestUtils.hasParam(request, "ajax") + && "executeFlowCycleFromExecutor".equals(HttpRequestUtils.getParam(request,"ajax"))){ + if (address != null && !this.isAddrUriAllowed(address.getHostString(), baseRequest.getMetaData().getURI().getDecodedPath())) { + logger.warn("Illegal access detected , ip >> {} , path >> {}",address.getHostString(),baseRequest.getMetaData().getURI()); + response.sendError(403); + baseRequest.setHandled(true); + return; + } + } + } + } + this.getHandler().handle(target, baseRequest, request, response); + } + }; + ipAccessHandler.setWhite(whiteListArr); + ipAccessHandler.setHandler(root); + server.setHandler(ipAccessHandler); + } + }catch (Exception e){ + logger.error("setting Executor whiteList failed ,caused by {}" , e); + } } public AlerterHolder getAlerterHolder() { diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/WebServerProvider.java b/azkaban-web-server/src/main/java/azkaban/webapp/WebServerProvider.java index 76bc699..47f0910 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/WebServerProvider.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/WebServerProvider.java @@ -23,8 +23,6 @@ import azkaban.utils.Props; import javax.inject.Inject; import com.google.inject.Provider; -import java.util.List; - //import org.mortbay.jetty.Connector; //import org.mortbay.jetty.Server; //import org.mortbay.jetty.bio.SocketConnector; @@ -40,6 +38,7 @@ public class WebServerProvider implements Provider { private static final Logger logger = LoggerFactory.getLogger(WebServerProvider.class); private static final int MAX_HEADER_BUFFER_SIZE = 10 * 1024 * 1024; + private static final boolean JETTY_SEND_SERVER_VERSION = false; @Inject private Props props; @@ -87,6 +86,7 @@ private ServerConnector createHttpConnector(Server server) { HttpConfiguration httpConfig = new HttpConfiguration(); setHeaderBufferSize(httpConfig); + setSendServerVersion(httpConfig); int port = this.props.getInt("jetty.port", Constants.DEFAULT_PORT_NUMBER); String bindAddress = this.props.getString("jetty.hostname", "0.0.0.0"); @@ -122,6 +122,11 @@ private void setHeaderBufferSize(HttpConfiguration configuration) { configuration.setRequestHeaderSize(MAX_HEADER_BUFFER_SIZE); } + private void setSendServerVersion(HttpConfiguration configuration) { + final boolean sendServerVersion = props.getBoolean("jetty.send.server.version", JETTY_SEND_SERVER_VERSION); + configuration.setSendServerVersion(sendServerVersion); + } + private ServerConnector createHttpsConnector(Server jettyServer) { SslContextFactory sslContextFactory = new SslContextFactory(); @@ -135,6 +140,7 @@ private ServerConnector createHttpsConnector(Server jettyServer) { HttpConfiguration httpConfig = new HttpConfiguration(); setHeaderBufferSize(httpConfig); + setSendServerVersion(httpConfig); httpConfig.addCustomizer(new SecureRequestCustomizer()); final int port = this.props.getInt("jetty.ssl.port", Constants.DEFAULT_SSL_PORT_NUMBER); diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ExecutorServlet.java b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ExecutorServlet.java index 5f317f3..d3a8e15 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ExecutorServlet.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ExecutorServlet.java @@ -458,8 +458,9 @@ private void ajaxUpdateQueueProcessor(final HttpServletRequest req, final HttpSe private void ajaxFetchscheduledflowgraphNew(final String projectName, final String flowName, final HashMap ret, final User user) throws ServletException { final Project project = getProjectAjaxByPermission(ret, projectName, user, Type.EXECUTE); + Map stringStringMap = loadExecutorServletI18nData(); if (project == null) { - ret.put("error", "Project '" + projectName + "' doesn't exist."); + ret.put("error", stringStringMap.get("permissionForAction") + projectName); return; } try { @@ -2359,10 +2360,6 @@ private Map repeatDateCompute(final JsonObject jsonObject) { List runDateTimeList = GsonUtils.jsonToJavaObject(jsonObject.getAsJsonArray("runDateTimeList"), new TypeToken>() {}.getType()); timeList.addAll(runDateTimeList); try { -// String month = getParam(req, "month"); -// String day = getParam(req, "day"); -// String hour = "0";//getParam(req, "hour"); -// String min = "0";//getParam(req, "min"); String recoverNum = jsonObject.get("recoverNum").getAsString(); String recoverInterval = jsonObject.get("recoverInterval").getAsString(); diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/HistoryServlet.java b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/HistoryServlet.java index 14d1632..67e1de3 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/HistoryServlet.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/HistoryServlet.java @@ -27,7 +27,6 @@ import azkaban.utils.Utils; import azkaban.utils.WebUtils; import azkaban.webapp.AzkabanWebServer; - import com.webank.wedatasphere.schedulis.common.i18nutils.LoadJsonUtils; import com.webank.wedatasphere.schedulis.common.jobExecutor.utils.SystemBuiltInParamJodeTimeUtils; import com.webank.wedatasphere.schedulis.common.system.SystemManager; diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java index 4dedff5..3811fe4 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java @@ -37,6 +37,7 @@ import com.webank.wedatasphere.schedulis.common.i18nutils.LoadJsonUtils; import com.webank.wedatasphere.schedulis.common.system.SystemManager; import com.webank.wedatasphere.schedulis.common.user.SystemUserManager; +import com.webank.wedatasphere.schedulis.common.utils.RSAUtils; import com.webank.wedatasphere.schedulis.common.utils.XSSFilterUtils; import java.io.BufferedInputStream; import java.io.File; @@ -359,8 +360,8 @@ private void handleDssRequest(final HttpServletRequest req, final HttpServletRes RedirectMsg redirectMsg = appJointAuth.getRedirectMsg(req); String redirectUrl = redirectMsg.getRedirectUrl(); String username = redirectMsg.getUser(); - logger.info("Succeed to get redirect url: {}, and username: {}" + redirectUrl + "," + username); - // 通过用户名，完成无密码登录（将用户信息写入HttpSession） + logger.info("Succeed to get redirect url: {}, and username: {}", redirectUrl, username); + // 通过用户名，在你的系统完成无密码登录（将用户信息写入HttpSession） if(session == null){ handleDssLoginAction(username, req,resp); } @@ -435,11 +436,36 @@ protected void doPost(final HttpServletRequest req, final HttpServletResponse re final String password = (String) params.get("userpwd"); final String ip = getRealClientIpAddr(req); - try { - session = createSession(username, password, ip); - } catch (final UserManagerException e) { - writeResponse(resp, "Login error: " + e.getMessage()); - return; + String wtss_secret_de = props.getString("dss.secret", ""); + String wtss_private_key = props.getString("wtss.private.key", ""); + String from_dss_secret_de = ""; + if(params.containsKey("dss_secret")){ + String from_dss_secret_en = (String)params.get("dss_secret"); + logger.info("handle dss login , secret > {}" , from_dss_secret_en); + try { + if(from_dss_secret_en!=null){ + from_dss_secret_en = from_dss_secret_en.replaceAll(" ","+"); + } + from_dss_secret_de = RSAUtils.decrypt(from_dss_secret_en,wtss_private_key); + } catch (Exception e) { + logger.error("parse dss.secret failed , caused by {} " , e); + } + } + if(wtss_secret_de.equals(from_dss_secret_de)){ + logger.info("handle dss login , dss_secret pass check" ); + try{ + session = createSession(username, password, ip, wtss_secret_de); + } catch(final Exception e){ + writeResponse(resp, "Login error: " + e.getMessage()); + return; + } + }else{ + try { + session = createSession(username, password, ip); + } catch (final UserManagerException e) { + writeResponse(resp, "Login error: " + e.getMessage()); + return; + } } } handleMultiformPost(req, resp, params, session); @@ -516,8 +542,61 @@ private Session createDssSession(final String username, final HttpServletRequest private Session createSession(final HttpServletRequest req) throws UserManagerException, ServletException, IOException { final String username = getParam(req, "username"); - final String password = getParam(req, "userpwd"); + String password = getParam(req, "userpwd"); + + final Props props = this.application.getServerProps(); + + if (hasParam(req, "encryption") && "true".equals(getParam(req, "encryption"))){ + String wtss_private_key = props.getString("wtss.private.key", ""); + logger.debug("encryption is enable , decode password {}" , password); + try { + if(password!=null){ + password = password.replaceAll(" ","+"); + } + password = RSAUtils.decrypt(password,wtss_private_key); + } catch (Exception e) { + throw new RuntimeException("parse encryption secret info failed , caused by {} " + e.getMessage()); + } + } + final String ip = getRealClientIpAddr(req); + + try{ + String wtss_secret_de = props.getString("dss.secret", ""); + String wtss_private_key = props.getString("wtss.private.key", ""); + String from_dss_secret_de = ""; + if(hasParam(req, "dss_secret")){ + String from_dss_secret_en = (String)getParam(req, "dss_secret"); + logger.debug("handle dss login , secret > {}" , from_dss_secret_en); + try { + if(from_dss_secret_en!=null){ + from_dss_secret_en = from_dss_secret_en.replaceAll(" ","+"); + } + from_dss_secret_de = RSAUtils.decrypt(from_dss_secret_en,wtss_private_key); + } catch (Exception e) { + throw new RuntimeException("parse dss.secret failed , caused by " , e); + } + } + + if(wtss_secret_de.equals(from_dss_secret_de)){ + logger.debug("handle dss login , dss_secret pass check" ); + //如果超级用户用户名和密码都是对的，那么我们直接放行 + if(!StringUtils.isFromBrowser(req.getHeader("User-Agent"))){ + logger.info("not browser."); + Session cacheSession = this.application.getSessionCache().getSessionByUsername(username); + if(cacheSession != null){ + logger.info("session not found."); + return cacheSession; + } + } + Session newSession = createSession(username, password, ip, wtss_secret_de); + getApplication().getSessionCache().addSession(newSession); + return newSession; + } + }catch(final Exception e){ + logger.error("no super user", e); + //没有超级用户，直接ignore + } return createSession(username, password, ip, req); } @@ -555,6 +634,26 @@ private Session createSession(final String username, final String password, fina return session; } + + private Session createSession(final String username, final String password, final String ip, + final String superUser) throws UserManagerException{ + + UserManager manager = getApplication().getTransitionService().getUserManager(); + if (manager instanceof SystemUserManager){ + //不改接口，直接改SystemUserManager，这样做到少侵入 + SystemUserManager userManager = (SystemUserManager)manager; + final User user = userManager.getUser(username, password, superUser); + logger.info("User is {}", user.toString()); + final String uuid = UUID.randomUUID().toString(); + return new Session(uuid, user, ip); + }else{ + logger.warn("user manager 不是 WebankXmlUserManager 实例，不能进行创建session"); + return null; + } + } + + + protected boolean hasPermission(final Project project, final User user, final Permission.Type type) { if (project.hasPermission(user, type)) { return true; @@ -600,6 +699,11 @@ protected void handleAjaxLoginAction(final HttpServletRequest req, ret.put("error", "Login in error. " + e.getMessage()); return; } + if (null == session){ + logger.error("session is null"); + ret.put("error","Login in error, session is null."); + return; + } final Cookie cookie = new Cookie(SESSION_ID_NAME, session.getSessionId()); cookie.setPath("/"); if("open".equals(nginxSSL)) { @@ -647,7 +751,7 @@ protected void handleDssLoginAction(String username,final HttpServletRequest req try { session = createDssSession(username,req); } catch (final UserManagerException | IOException e) { - logger.error("Dss Login in error. " + e.getMessage()); + logger.error("Dss Login in error.", e); return; } if (null == session){ diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java index 8447ceb..0c2cba7 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java @@ -420,6 +420,9 @@ private void handleAJAXAction(final HttpServletRequest req, }else if (ajaxName.equals("checkRunningPageKillFlowPermission")) { // 检查用户Kill运行中的工作流权限 ajaxCheckRunningPageKillFlowPermission(req, resp, ret, session); + } else if (ajaxName.equals("checkUserSwitchScheduleFlowPermission")) { + // 检查用户开启或关闭定时调度权限 + ajaxcheckUserSwitchScheduleFlowPermission(req, resp, ret, session); } else { ret.put("error", "Cannot execute command " + ajaxName); } @@ -700,6 +703,7 @@ private void ajaxCheckUserDeleteScheduleFlowPermission(HttpServletRequest req, H final User user = session.getUser(); if (wtss_project_privilege_check) { int deleteScheduleFlowFlag = checkUserOperatorFlag(user); + resultMap.put("deleteScheduleFlowFlag", deleteScheduleFlowFlag); logger.info("current user delete schedule flow permission flag is deleteScheduleFlowFlag=" + deleteScheduleFlowFlag); } else { resultMap.put("deleteScheduleFlowFlag", 1); @@ -892,6 +896,59 @@ private void ajaxCheckDeleteScheduleInDescriptionFlagPermission(HttpServletReque } + /** + * 检查用户开启或关闭定时调度权限 + * + * @param req + * @param resp + * @param resultMap + * @param session wtss_project_privilege_check + */ + private void ajaxcheckUserSwitchScheduleFlowPermission(HttpServletRequest req, HttpServletResponse resp, + HashMap resultMap, Session session) { + + try { + if (session != null) { + final String projectName = getParam(req, "project"); + final User user = session.getUser(); + final Project project = getProjectAjaxByPermission(resultMap, projectName, user, Type.SCHEDULE); + Map stringStringMap = loadProjectManagerServletI18nData(); + if (project == null) { + resultMap.put("error", stringStringMap.get("permissionForAction") + projectName); + resultMap.put("switchScheduleFlowFlag", 3); + return; + } + if (wtss_project_privilege_check) { + int switchScheduleFlowFlag = checkUserOperatorFlag(user); + resultMap.put("switchScheduleFlowFlag", switchScheduleFlowFlag); + logger.info("current user active schedule flow permission flag is switchScheduleFlowFlag=" + switchScheduleFlowFlag); + } else { + resultMap.put("switchScheduleFlowFlag", 1); + } + } + } catch (Exception e) { + logger.error("Failed to find current user active schedule flow flow permission flag, caused by:{}", e); + } + } + + protected Project getProjectAjaxByPermission(final Map ret, final String projectName, + final User user, final Permission.Type type) { + final Project project = this.projectManager.getProject(projectName); + + Map dataMap = loadProjectManagerServletI18nData(); + + if (project == null) { + ret.put("error", dataMap.get("project") + projectName + dataMap.get("notExist")); + } else if (!hasPermission(project, user, type)) { + ret.put("error", "User " + user.getUserId() + " doesn't have " + project.getName() + " of " + type.name() + + " permissions, please contact with the project creator."); + } else { + return project; + } + + return null; + } + /** * 检查用户KILL正在运行页面flow权限 * diff --git a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ScheduleServlet.java b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ScheduleServlet.java index e4d2036..18d5ce8 100644 --- a/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ScheduleServlet.java +++ b/azkaban-web-server/src/main/java/azkaban/webapp/servlet/ScheduleServlet.java @@ -1022,6 +1022,8 @@ private boolean scheduleAllFlow(Project project, Flow flow, Map otherOptions.put("successAlertLevel", json.get("successAlertLevel").getAsString()); } + otherOptions.put("activeFlag", true); + try { //设置告警用户部门信息 String userDep = transitionService.getUserDepartmentByUsername(user.getUserId()); diff --git a/azkaban-web-server/src/main/java/com/webank/wedatasphere/schedulis/web/webapp/LocaleFilter.java b/azkaban-web-server/src/main/java/com/webank/wedatasphere/schedulis/web/webapp/LocaleFilter.java index f6e1ed7..eb5a990 100644 --- a/azkaban-web-server/src/main/java/com/webank/wedatasphere/schedulis/web/webapp/LocaleFilter.java +++ b/azkaban-web-server/src/main/java/com/webank/wedatasphere/schedulis/web/webapp/LocaleFilter.java @@ -89,7 +89,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo req.getSession().setAttribute("TRANS_I18N_LOCALE", languageType); LoadJsonUtils.setLanguageType(languageType); } catch (Exception e) { - logger.error("a fatal error had happen when init locale"); + logger.error("a fatal error had happen when init locale languageType, caused by:" , e); LoadJsonUtils.setLanguageType("zh_CN"); } } diff --git a/azkaban-web-server/src/web/js/azkaban/util/schedule.js b/azkaban-web-server/src/web/js/azkaban/util/schedule.js index 6d9d5d7..9b43513 100644 --- a/azkaban-web-server/src/web/js/azkaban/util/schedule.js +++ b/azkaban-web-server/src/web/js/azkaban/util/schedule.js @@ -105,6 +105,56 @@ function editScheduleClick(scheduleId, projectName, flowName, cronExpression) { } +// 定时调度页面, 定时调度工作流列表, 对显示的调度任务点击调度开启关闭 +function switchScheduleClick (index, scheduleId, projectName, flowName, cronExpression) { + + // 需要校验是否具有修改项目调度权限 1:允许, 2:不允许 + var requestURL = "/manager?ajax=checkUserSwitchScheduleFlowPermission&project=" + projectName; + $.ajax({ + url: requestURL, + type: "get", + async: false, + dataType: "json", + success: function (data) { + if (data["switchScheduleFlowFlag"] == 1) { + console.log("click switch schedule button."); + var currentActiveFlag = document.getElementById("schedules-tbody").rows[index].cells[8].innerHTML; + console.log("currentActiveFlag=" + currentActiveFlag); + var destActiveFlag = false; + if (currentActiveFlag == "false") { + destActiveFlag = true; + } + + var scheduleActiveData = { + scheduleId: scheduleId, + ajax: "setScheduleActiveFlag", + activeFlag: destActiveFlag + }; + + var scheduleURL = "/schedule" + var successHandler = function (data) { + if (data.error) { + alert(data.error); + } else { + // 触发变更就行, 不是刷新所有页面 + scheduleListView.handlePageChange(); + } + }; + $.post(scheduleURL, scheduleActiveData, successHandler, "json"); + } else if (data["switchScheduleFlowFlag"] == 2) { + $('#user-operator-schedule-flow-permit-panel').modal(); + $('#title-user-operator-schedule-flow-permit').text(wtssI18n.view.scheduleActivePermission); + $('#body-user-operator-schedule-flow-permit').html(wtssI18n.view.noScheSwitchConfigPermission); + } else if(data["switchScheduleFlowFlag"] == 3){ + $('#user-operator-schedule-flow-permit-panel').modal(); + $('#title-user-operator-schedule-flow-permit').text(wtssI18n.view.scheduleActivePermission); + $('#body-user-operator-schedule-flow-permit').html(data.error); + } + } + }); + +} + $(function () { scheduleView = new azkaban.ScheduleView({ diff --git a/azkaban-web-server/src/web/js/azkaban/view/schedule-flow-edit-dialog.js b/azkaban-web-server/src/web/js/azkaban/view/schedule-flow-edit-dialog.js index 3582829..9f03006 100644 --- a/azkaban-web-server/src/web/js/azkaban/view/schedule-flow-edit-dialog.js +++ b/azkaban-web-server/src/web/js/azkaban/view/schedule-flow-edit-dialog.js @@ -302,29 +302,35 @@ azkaban.FlowScheduleDialogView = Backbone.View.extend({ }; var self = this; var successHandler = function (data) { - console.log("data fetched"); - graphModel.addFlow(data); + if (data.error) { + $('#user-operator-schedule-flow-permit-panel').modal(); + $('#title-user-operator-schedule-flow-permit').text(wtssI18n.view.scheduleConfigPermission); + $('#body-user-operator-schedule-flow-permit').html(data.error); + } else { + console.log("data fetched"); + graphModel.addFlow(data); - if (exgraph) { - self.assignInitialStatus(data, exgraph); - } + if (exgraph) { + self.assignInitialStatus(data, exgraph); + } - // Auto disable jobs that are finished. - disableFinishedJobs(data); - executingSvgGraphView = new azkaban.SvgGraphView({ - el: $('#schedule-flow-executing-graph'), - model: graphModel, - render: false, - rightClick: { - "node": expanelNodeClickCallback, - "edge": expanelEdgeClickCallback, - "graph": expanelGraphClickCallback - }, - tooltipcontainer: "#schedule-svg-div-custom" - }); + // Auto disable jobs that are finished. + disableFinishedJobs(data); + executingSvgGraphView = new azkaban.SvgGraphView({ + el: $('#schedule-flow-executing-graph'), + model: graphModel, + render: false, + rightClick: { + "node": expanelNodeClickCallback, + "edge": expanelEdgeClickCallback, + "graph": expanelGraphClickCallback + }, + tooltipcontainer: "#schedule-svg-div-custom" + }); - if (callback) { - callback.call(this); + if (callback) { + callback.call(this); + } } }; $.get(requestURL, requestData, successHandler, "json"); diff --git a/bin/config/wtss-web/web_plugin_system.properties b/bin/config/wtss-web/web_plugin_system.properties index 8963828..ff20b08 100644 --- a/bin/config/wtss-web/web_plugin_system.properties +++ b/bin/config/wtss-web/web_plugin_system.properties @@ -7,6 +7,9 @@ viewer.servlet.class=azkaban.viewer.system.SystemServlet wtss.db.jdbc.url=[#SYSTEM_DB_URL] wtss.db.username=[#SYSTEM_DB_USERNAME] wtss.db.password=[#SYSTEM_DB_PASSWORD] +esb.app.domain=[#ESB_APP_URL] +esb.app.id= +esb.app.token= diff --git a/pom.xml b/pom.xml index 6508370..f755685 100755 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 2.0.1 3.2.2 4.1 - 1.16.1 + 1.20 1.3.1 2.1.1 1.5 @@ -49,31 +49,34 @@ 28.1-jre 1.4.193 4.5.3 - 0.10.0.0 3.0 2.10.0 2.14.0 - 5.1.48 + 8.0.18 3.1.0 1.4.5 - 9.4.24.v20191120 + 9.4.30.v20200611 1.9.13 4.3 2.0 4.3 3.0.2 0.11.0 - 1.7.18 + 1.7.30 1.15.7 - 0.10.0.0 2.2.1 1.18 1.6.1 1.7 2.0 3.0.0 + 3.0.0 2.13.3 - 0.0.4-SNAPSHOT + 4.1.2 + 0.9.5.5 + 1.9.4 + 2.11.2 + 0.7.0 @@ -82,6 +85,7 @@ gson ${gson.version} + org.apache.logging.log4j log4j-core @@ -139,12 +143,17 @@ ${awaitility.version} test + + org.forgerock.opendj + opendj-core + ${opendjcore.version} + + com.jcraft jsch 0.1.55 - commons-collections commons-collections @@ -160,12 +169,45 @@ commons-lang ${common.lang.version} - org.apache.commons commons-lang3 ${commons.lang3.version} + + io.jsonwebtoken + jjwt + 0.9.1 + + + jackson-annotations + com.fasterxml.jackson.core + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} +