From 21c10f886288ee6667f052eff294da7964de8e64 Mon Sep 17 00:00:00 2001 From: muzhongjiang Date: Thu, 23 Apr 2020 17:12:16 +0800 Subject: [PATCH 1/2] fixed bug #2492 (#2497) Co-authored-by: mzjnumber1@163.com Co-authored-by: dailidong --- script/dolphinscheduler-daemon.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 19669e5ede..b0437a30c2 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -41,7 +41,7 @@ export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` -export DOLPHINSCHEDULER_PID_DIR=/tmp/ +export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* @@ -54,7 +54,7 @@ if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then fi log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out -pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid +pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME From f2bfdc14f7abfb8ce5a886f5b796056330eaa569 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 23 Apr 2020 17:37:04 +0800 Subject: [PATCH 2/2] #2499 bug fix (#2505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix * host and workergroup compatible * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * #2499 bug fix Co-authored-by: qiaozhanwei --- .../src/test/resources/alert.properties | 67 ------------------- .../server/master/MasterServer.java | 7 +- .../server/zk/ZKMasterClient.java | 12 +++- 3 files changed, 15 insertions(+), 71 deletions(-) delete mode 100644 dolphinscheduler-alert/src/test/resources/alert.properties diff --git a/dolphinscheduler-alert/src/test/resources/alert.properties b/dolphinscheduler-alert/src/test/resources/alert.properties deleted file mode 100644 index ce233cea37..0000000000 --- a/dolphinscheduler-alert/src/test/resources/alert.properties +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# For unit test - -#alert type is EMAIL/SMS -alert.type=EMAIL - -# mail server configuration -mail.protocol=SMTP -mail.server.host=xxx.xxx.test -mail.server.port=25 -mail.sender=xxx@xxx.com -mail.user=xxx@xxx.com -mail.passwd=111111 - -# Test double -test.server.factor=3.0 - - -# Test NumberFormat -test.server.testnumber=abc - -# Test array -test.server.list=xxx.xxx.test1,xxx.xxx.test2,xxx.xxx.test3 - -# Test enum -test.server.enum1=MASTER -test.server.enum2=DEAD_SERVER -test.server.enum3=abc - -# TLS -mail.smtp.starttls.enable=true -# SSL -mail.smtp.ssl.enable=false -mail.smtp.ssl.trust=xxx.xxx.com - -#xls file path,need create if not exist -xls.file.path=/tmp/xls - -# Enterprise WeChat configuration -enterprise.wechat.enable=false -enterprise.wechat.corp.id=xxxxxxx -enterprise.wechat.secret=xxxxxxx -enterprise.wechat.agent.id=xxxxxxx -enterprise.wechat.users=xxxxxxx -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} - - - diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6df82b5a60..d86374244f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -117,9 +117,12 @@ public class MasterServer { this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_RESPONSE, new TaskKillResponseProcessor()); this.nettyRemotingServer.start(); - // - this.zkMasterClient.start(); + // register this.masterRegistry.registry(); + + // self tolerant + this.zkMasterClient.start(); + // masterSchedulerService.start(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 46f48b6d76..e923157360 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -40,6 +41,8 @@ import org.springframework.stereotype.Component; import java.util.Date; import java.util.List; +import static org.apache.dolphinscheduler.common.Constants.*; + /** * zookeeper master client @@ -72,8 +75,13 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // check if fault tolerance is required?failure and tolerance - if (getActiveMasterNum() == 1 && checkZKNodeExists(OSUtils.getHost(), ZKNodeType.MASTER)) { + while (!checkZKNodeExists(OSUtils.getHost(), ZKNodeType.MASTER)){ + ThreadUtils.sleep(SLEEP_TIME_MILLIS); + } + + + // self tolerant + if (getActiveMasterNum() == 1) { failoverWorker(null, true); failoverMaster(null); }