From cfd3266ca24c4c287629495dff02f88f9f1323c7 Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Sat, 28 Dec 2019 21:35:10 +0800 Subject: [PATCH 1/3] we should insert alert DB once , and trigger this type of alert 3 times --- .../apache/dolphinscheduler/server/master/MasterServer.java | 6 ++---- .../apache/dolphinscheduler/server/worker/WorkerServer.java | 4 +--- .../apache/dolphinscheduler/server/zk/ZKMasterClient.java | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index e8c8b6779e..1f5b16a25a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -152,10 +152,8 @@ public class MasterServer implements IStoppable { @Override public void run() { if (zkMasterClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index fe9c4991ac..ea8330e18a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable { public void run() { // worker server exit alert if (zkWorkerClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index a26a217665..1c7b737f5b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient { */ private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { - String serverType = zkNodeType.toString(); - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, serverHost, serverType); - } + String serverType = zkNodeType.toString(); + alertDao.sendServerStopedAlert(1, serverHost, serverType); } /** From b053322466761354b56bb760b3a32d4d127a60a9 Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Sun, 29 Dec 2019 16:54:40 +0800 Subject: [PATCH 2/3] refactor AbstractZKClient --- .../common/zk/AbstractZKClient.java | 96 +++++++------------ 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index 0e95dddb36..c3ba718270 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -16,6 +16,22 @@ */ package org.apache.dolphinscheduler.common.zk; +import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX; +import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; +import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.ZKNodeType; @@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; - /** * abstract zookeeper client @@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ return; } - byte[] bytes = zkClient.getData().forPath(znode); - String resInfoStr = new String(bytes); + String resInfoStr = super.get(znode); String[] splits = resInfoStr.split(Constants.COMMA); if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; @@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(zNode) == null || - zkClient.checkExists().forPath(deadServerPath) != null ){ + if(!isExisted(zNode) || isExisted(deadServerPath)){ return true; } @@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ public void removeDeadServerByHost(String host, String serverType) throws Exception { - List deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath()); + List deadServers = super.getChildrenKeys(getDeadZNodeParentPath()); for(String serverPath : deadServers){ if(serverPath.startsWith(serverType+UNDERLINE+host)){ - String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; - if(zkClient.checkExists().forPath(server) != null){ - zkClient.delete().forPath(server); - logger.info("{} server {} deleted from zk dead server path success" , serverType , host); - } + String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; + super.remove(server); + logger.info("{} server {} deleted from zk dead server path success" , serverType , host); } } } @@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ // create temporary sequence nodes for master znode String parentPath = getZNodeParentPath(zkNodeType); String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); - String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( - serverPathPrefix + UNDERLINE, heartbeatZKInfo.getBytes()); + String registerPath = serverPathPrefix + UNDERLINE; + super.persistEphemeral(registerPath, heartbeatZKInfo); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); return registerPath; } @@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ } registerPath = createZNodePath(zkNodeType); - // handle dead server + // handle dead server handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP); return registerPath; @@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ }else if(opType.equals(ADD_ZK_OP)){ String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(deadServerPath) == null){ + if(!super.isExisted(deadServerPath)){ //add dead server info to zk dead server path : /dead-servers/ - zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); + super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo)); logger.info("{} server dead , and {} added to zk dead server path success" , zkNodeType.toString(), zNode); @@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ List childrenList = new ArrayList<>(); try { // read master node parent path from conf - if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){ - childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER)); + if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){ + childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER)); } } catch (Exception e) { - if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ - logger.error("zookeeper service not started",e); - }else{ - logger.error(e.getMessage(),e); - } - - }finally { - return childrenList.size(); + logger.error("getActiveMasterNum error",e); } + return childrenList.size(); } /** @@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ Map masterMap = new HashMap<>(); try { String path = getZNodeParentPath(zkNodeType); - List serverList = getZkClient().getChildren().forPath(path); + List serverList = super.getChildrenKeys(path); for(String server : serverList){ - byte[] bytes = getZkClient().getData().forPath(path + "/" + server); - masterMap.putIfAbsent(server, new String(bytes)); + masterMap.putIfAbsent(server, super.get(path + "/" + server)); } } catch (Exception e) { logger.error("get server list failed : " + e.getMessage(), e); @@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ */ protected void initSystemZNode(){ try { - createNodePath(getMasterZNodeParentPath()); - createNodePath(getWorkerZNodeParentPath()); - createNodePath(getDeadZNodeParentPath()); + persist(getMasterZNodeParentPath(), ""); + persist(getWorkerZNodeParentPath(), ""); + persist(getDeadZNodeParentPath(), ""); } catch (Exception e) { logger.error("init system znode failed : " + e.getMessage(),e); } } - /** - * create zookeeper node path if not exists - * @param zNodeParentPath zookeeper parent path - * @throws Exception errors - */ - private void createNodePath(String zNodeParentPath) throws Exception { - if(null == zkClient.checkExists().forPath(zNodeParentPath)){ - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath); - } - } - /** * server self dead, stop all threads * @param serverHost server host From cc786e6d76b5eb208bef7a23872cca6c2fb60920 Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Mon, 30 Dec 2019 18:18:18 +0800 Subject: [PATCH 3/3] remove swagger-annotations --- dolphinscheduler-dao/pom.xml | 6 ------ .../java/org/apache/dolphinscheduler/dao/entity/User.java | 5 ----- 2 files changed, 11 deletions(-) diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 85467e12d4..af1cdb8b78 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -162,12 +162,6 @@ spring-test test - - io.swagger - swagger-annotations - 1.5.20 - compile - org.yaml snakeyaml diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index c4df0679ef..b6def18922 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -22,8 +22,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; @@ -33,7 +31,6 @@ import java.util.Date; */ @Data @TableName("t_ds_user") -@ApiModel(description = "UserModelDesc") public class User { /** @@ -45,13 +42,11 @@ public class User { /** * user name */ - @ApiModelProperty(name = "userName", notes = "USER_NAME",dataType = "String",required = true) private String userName; /** * user password */ - @ApiModelProperty(name = "userPassword", notes = "USER_PASSWORD",dataType = "String",required = true) private String userPassword; /**