opt start script

azkaban-common/pom.xml

@@ -212,18 +212,14 @@
       <version>${dropwizard.core.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.forgerock.opendj</groupId>
+      <groupId>org.openidentityplatform.opendj</groupId>
       <artifactId>opendj-grizzly</artifactId>
-      <version>3.0.0</version>
-      <scope>system</scope>
-      <systemPath>${project.basedir}/extlib/opendj-grizzly-3.0.0.jar</systemPath>
+      <version>${opendjcore.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.forgerock.opendj</groupId>
+      <groupId>org.openidentityplatform.opendj</groupId>
       <artifactId>opendj-core</artifactId>
-      <version>3.0.0</version>
-      <scope>system</scope>
-      <systemPath>${project.basedir}/extlib/opendj-core-3.0.0.jar</systemPath>
+      <version>${opendjcore.version}</version>
     </dependency>
     <dependency>
       <groupId>org.yaml</groupId>

azkaban-common/src/test/java/azkaban/db/AzDBTestUtility.java

@@ -56,7 +56,7 @@ public static class EmbeddedMysqlDataSource extends AzkabanDataSource {
 
     public EmbeddedMysqlDataSource() {
       super();
-      final String url = "jdbc:mysql://127.0.0.1:3306/database?useUnicode=true&characterEncoding=UTF-8";
+      final String url = "jdbc:mysql://10.255.0.76:3306/wtss_qyh_test?useUnicode=true&characterEncoding=UTF-8";
       setDriverClassName("com.mysql.jdbc.Driver");
       setUrl(url);
       setUsername("root");

azkaban-db/src/test/java/azkaban/db/AzDBTestUtility.java

@@ -56,7 +56,7 @@ public static class EmbeddedMysqlDataSource extends AzkabanDataSource {
 
     public EmbeddedMysqlDataSource() {
       super();
-      final String url = "jdbc:mysql://localhost:port/database?useUnicode=true&characterEncoding=UTF-8";
+      final String url = "jdbc:mysql://localhost:port/bdp_scheduler_01?useUnicode=true&characterEncoding=UTF-8";
       setDriverClassName("com.mysql.jdbc.Driver");
       setUrl(url);
       setUsername("username");

azkaban-exec-server/src/main/bash/internal/internal-start-executor.sh

@@ -96,7 +96,15 @@ function start(){
   LOG INFO "Starting AzkabanExecutorServer on port $executorport ..."
   java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.execapp.AzkabanExecutorServer -conf $conf $@ >> $logFile 2>&1 &
   echo $! > $azkaban_dir/currentpid
-  LOG INFO "AzkabanExecutorServer started successfully."
+  sleep 3s
+  processName=`jps|grep AzkabanExecutorServer`
+  if [ ! -n "$processName" ]
+  then
+      LOG INFO "AzkabanExecutorServer startup failed"
+      return 1
+  else
+      return 0
+  fi
 }
 
 function genServerId(){
@@ -121,43 +129,35 @@ function updataExecutorStatus(){
   LOG INFO "start update executor status..."
   LOCAL_HOSTNAME="`hostname`"
   local azkabanConf=$azkaban_dir/conf/azkaban.properties
-  HOSTNAME=`getValue "mysql.host" $azkabanConf`
-  PORT=`getValue "mysql.port" $azkabanConf`
-  DATABASE=`getValue "mysql.database" $azkabanConf`
-  USERNAME=`getValue "mysql.user" $azkabanConf`
-  PASSWORD=`getValue "mysql.password" $azkabanConf "mysql.password=" | base64 -d`
-
-  select_sql="select count(*) as num from executors where host='${LOCAL_HOSTNAME}'"
+  EXECUTOR_PORT=`getValue "executor.port" $azkabanConf`
   start_finish=0
   runtime=0
   while [[ $start_finish != 1 ]]; do
-      start_finish=`mysql -h${HOSTNAME} -P${PORT} -u${USERNAME} -p${PASSWORD} ${DATABASE} -e "${select_sql}" --default-character-set=UTF8`
-      start_finish=${start_finish[0]:4}
-      LOG INFO " 更新数据库执行节点记录成功数, 0 失败 1 成功。" 
-      LOG INFO " 执行结果: ${start_finish}"
-      sleep 5s
+      result=`curl -POST http://${LOCAL_HOSTNAME}:${EXECUTOR_PORT}/executor -d action=activate`
+      LOG INFO " exectue result: ${result}"
+      [[ "${result}" =~ .*success.* ]] && { break; }
+      sleep 3s
       runtime=$(( $runtime + 1 ))
-      LOG INFO " 当前运行次数为： ${runtime}  超过30次将退出。"
-      if [ ${runtime} -gt 30 ] 
+      LOG INFO "It has been run： ${runtime}  and will exit after 10 times。"
+      if [ ${runtime} -gt 10 ]
       then
           LOG ERROR "update executor status time out."
           return 1
       fi
   done
-  LOG INFO " 执行节点启动成功，更新数据库状态。"
-  update_sql="update executors set active=1 where host='${LOCAL_HOSTNAME}'"
-  mysql -h${HOSTNAME} -P${PORT} -u${USERNAME} -p${PASSWORD} ${DATABASE} -e "${update_sql}" --default-character-set=UTF8 
-  ret=$?
-  [ "$ret" != "0" ] && { LOG ERROR "update executor status failed. "; return 1; }
   LOG INFO "update executor success."
+  LOG INFO "AzkabanExecutorServer started successfully."
 }
 
 function preCheck(){
   LOG INFO "checking AzkabanExecutorServer status..."
-  if [ -f $azkaban_dir/currentpid ]
+  processName=`jps|grep AzkabanExecutorServer`
+  if [ -n "$processName" ]
   then
-    LOG INFO "AzkabanExecutorServer already started."
-    return 1
+      LOG INFO "AzkabanExecutorServer already started."
+      return 1
+  else
+      return 0
   fi
 }
 

azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/EventChecker.java

@@ -225,7 +225,7 @@ public static void main(String[] args) {
 //		p.put("msg.body","msg.body");
 
 		p.put("msg.eventchecker.jdo.option.name","msg");
-		p.put("msg.eventchecker.jdo.option.url","jdbc:mysql://locahost:port/database?useUnicode=true&characterEncoding=UTF-8");
+		p.put("msg.eventchecker.jdo.option.url","jdbc:mysql://locahost:port/wtss_qyh_test?useUnicode=true&characterEncoding=UTF-8");
 		p.put("msg.eventchecker.jdo.option.username","username");
 		p.put("msg.eventchecker.jdo.option.password","password");
 

azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/HttpUtils.java

@@ -203,7 +203,7 @@ public static void main(String[] args) {
     RequestBody requestBody = new FormBody.Builder()
         .add("targetDb", "bdp_test_ods_mask")
         .add("targetTable", "ccpd_dump")
-        .add("partition", "dcn_id=UA0/type_id=6042/ip=127.0.0.1/ds=2015-04-06")
+        .add("partition", "dcn_id=UA0/type_id=6042/ip=10.240.228.31/ds=2015-04-06")
         .build();
 
     FormBody.Builder formBuilder = new FormBody.Builder();

azkaban-jobtype/src/test/java/azkaban/jobtype/TestHadoopJobUtilsFindApplicationIdFromLog.java

@@ -102,7 +102,7 @@ public void testLogWithMultipleApplicationIdsAppearingMultipleTimes() throws IOE
     bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO See http://eat1-nertzwp02.grid.linkedin.com:8080/proxy/application_1440264346270_3044/ for details.\n");
     bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO Running job: job_1440264346270_3044\n");
     bw.write("28-08-2015 12:30:21 PDT Training_clickSelectFeatures INFO - INFO Closing idle connection Socket[addr=eat1-hcl5481.grid.linkedin.com/172.20.138.228,port=42492,localport=42382] to server eat1-hcl5481.grid.linkedin.com/172.20.138.228:42492\n");
-    bw.write("28-08-2015 12:30:37 PDT Training_clickSelectFeatures INFO - INFO Closing idle connection Socket[addr=eat1-nertznn01.grid.linkedin.com/127.0.0.1,port=9000,localport=30453] to server eat1-nertznn01.grid.linkedin.com/127.0.0.1:9000\n");
+    bw.write("28-08-2015 12:30:37 PDT Training_clickSelectFeatures INFO - INFO Closing idle connection Socket[addr=eat1-nertznn01.grid.linkedin.com/172.20.158.57,port=9000,localport=30453] to server eat1-nertznn01.grid.linkedin.com/172.20.158.57:9000\n");
     bw.write("28-08-2015 12:31:09 PDT Training_clickSelectFeatures INFO - INFO Job job_1440264346270_3044 running in uber mode : false\n");
     bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO Submitted application application_1440264346270_3088\n");
     bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO The url to track the job: http://eat1-nertzwp02.grid.linkedin.com:8080/proxy/application_1440264346270_3088/\n");

azkaban-web-server/src/bin/internal/internal-start-web.sh

@@ -100,7 +100,7 @@ function javaOption(){
 
 function start(){
     LOG INFO "starting AzkabanWebServer..."
-    java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $conf $cycle_stop $@ >> $logFile 2>&1 &
+    java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $conf $@ >> $logFile 2>&1 &
     echo $! > $azkaban_dir/currentpid
     sleep 3s
     processName=`jps|grep AzkabanWebServer`

azkaban-web-server/src/main/bash/internal/internal-start-web.sh

@@ -27,11 +27,14 @@ logFile=/appcom/logs/azkaban/webServerLog_`date +%F+%T`.out
 
 
 function preCheck(){
-  LOG INFO "checking AzkabanWebServer process status..."
-  if [ -f $azkaban_dir/currentpid ]
+  LOG INFO "checking AzkabanWebServer status..."
+  processName=`jps|grep AzkabanWebServer`
+  if [ -n "$processName" ]
   then
-    LOG INFO "AzkabanWebServer already started."
-    return 1
+      LOG INFO "AzkabanWebServer already started."
+      return 1
+  else
+      return 0
   fi
 }
 
@@ -80,26 +83,34 @@ function javaOption(){
     AZKABAN_OPTS="-Xmx16G -Xloggc:/appcom/logs/azkaban/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintHeapAtGC"
   fi
   # Set the log4j configuration file
-  if [ -f $conf/log4j.properties ]; then
-    AZKABAN_OPTS="$AZKABAN_OPTS -Dlog4j.configuration=file:$conf/log4j.properties -Dlog4j.log.dir=/appcom/logs/azkaban"
+  if [ -f $conf/log4j2.xml ]; then
+    AZKABAN_OPTS="$AZKABAN_OPTS -Dlog4j.configurationFile=$conf/log4j2.xml"
   else
     LOG ERROR "$conf/log4j.properties file doesn't exist."
     return 1
   fi
 
   executorport=`cat $conf/azkaban.properties | grep executor.port | awk -F '=' '{print($NF)}'`
-  
+
   AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$azkaban_dir"
 
   #AZKABAN_OPTS="$AZKABAN_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
-  
+
 }
 
 function start(){
     LOG INFO "starting AzkabanWebServer..."
-    java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $conf $cycle_stop $@ >> $logFile 2>&1 &
+    java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $conf $@ >> $logFile 2>&1 &
     echo $! > $azkaban_dir/currentpid
-    LOG INFO "AzkabanWebServer started successfully."
+    sleep 3s
+    processName=`jps|grep AzkabanWebServer`
+    if [ ! -n "$processName" ]
+    then
+        LOG INFO "AzkabanWebServer startup failed"
+        return 1
+    else
+        return 0
+    fi
 }
 
 function LOG(){