@ -17,36 +17,49 @@
package org.apache.dolphinscheduler.server.registry ; package org.apache.dolphinscheduler.server.registry ;
import org.apache.commons.collections.CollectionUtils ; import org.apache.dolphinscheduler.common.Constants ;
import org.apache.curator.framework.CuratorFramework ; import org.apache.dolphinscheduler.common.enums.ZKNodeType ;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent ;
import org.apache.dolphinscheduler.common.utils.StringUtils ; import org.apache.dolphinscheduler.common.utils.StringUtils ;
import org.apache.dolphinscheduler.dao.AlertDao ; import org.apache.dolphinscheduler.dao.AlertDao ;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup ;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper ;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory ;
import org.apache.dolphinscheduler.service.zk.AbstractListener ; import org.apache.dolphinscheduler.service.zk.AbstractListener ;
import org.slf4j.Logger ; import org.apache.dolphinscheduler.service.zk.AbstractZKClient ;
import org.slf4j.LoggerFactory ;
import org.springframework.beans.factory.InitializingBean ; import org.apache.commons.collections.CollectionUtils ;
import org.springframework.beans.factory.annotation.Autowired ; import org.apache.curator.framework.CuratorFramework ;
import org.springframework.stereotype.Service ; import org.apache.curator.framework.recipes.cache.TreeCacheEvent ;
import java.util.Collections ; import java.util.Collections ;
import java.util.HashMap ;
import java.util.HashSet ; import java.util.HashSet ;
import java.util.List ;
import java.util.Map ; import java.util.Map ;
import java.util.Set ; import java.util.Set ;
import java.util.concurrent.ConcurrentHashMap ; import java.util.concurrent.ConcurrentHashMap ;
import java.util.concurrent.Executors ;
import java.util.concurrent.ScheduledExecutorService ;
import java.util.concurrent.TimeUnit ;
import java.util.concurrent.locks.Lock ; import java.util.concurrent.locks.Lock ;
import java.util.concurrent.locks.ReentrantLock ; import java.util.concurrent.locks.ReentrantLock ;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP ; import javax.annotation.PreDestroy ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import org.springframework.beans.factory.InitializingBean ;
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.stereotype.Component ;
import org.springframework.stereotype.Service ;
/ * * / * *
* zookeeper node manager * serv ernode manager
* / * /
@Service @Service
public class ZookeeperNodeManager implements InitializingBean { public class Serv erNodeManagerimplements InitializingBean {
private final Logger logger = LoggerFactory . getLogger ( Zookeep erNodeManager. class ) ; private final Logger logger = LoggerFactory . getLogger ( Serv erNodeManager. class ) ;
/ * * / * *
* master lock * master lock
@ -58,6 +71,11 @@ public class ZookeeperNodeManager implements InitializingBean {
* / * /
private final Lock workerGroupLock = new ReentrantLock ( ) ; private final Lock workerGroupLock = new ReentrantLock ( ) ;
/ * *
* worker node info lock
* /
private final Lock workerNodeInfoLock = new ReentrantLock ( ) ;
/ * * / * *
* worker group nodes * worker group nodes
* / * /
@ -68,12 +86,34 @@ public class ZookeeperNodeManager implements InitializingBean {
* / * /
private final Set < String > masterNodes = new HashSet < > ( ) ; private final Set < String > masterNodes = new HashSet < > ( ) ;
/ * *
* worker node info
* /
private final Map < String , String > workerNodeInfo = new HashMap < > ( ) ;
/ * *
* executor service
* /
private ScheduledExecutorService executorService ;
/ * *
* zk client
* /
@Autowired
private ZKClient zkClient ;
/ * * / * *
* zookeeper registry center * zookeeper registry center
* / * /
@Autowired @Autowired
private ZookeeperRegistryCenter registryCenter ; private ZookeeperRegistryCenter registryCenter ;
/ * *
* worker group mapper
* /
@Autowired
private WorkerGroupMapper workerGroupMapper ;
/ * * / * *
* alert dao * alert dao
* / * /
@ -90,6 +130,11 @@ public class ZookeeperNodeManager implements InitializingBean {
* load nodes from zookeeper * load nodes from zookeeper
* / * /
load ( ) ; load ( ) ;
/ * *
* init executor service
* /
executorService = Executors . newSingleThreadScheduledExecutor ( new NamedThreadFactory ( "ServerNodeManagerExecutor" ) ) ;
executorService . scheduleWithFixedDelay ( new WorkerNodeInfoAndGroupDbSyncTask ( ) , 0 , 10 , TimeUnit . SECONDS ) ;
/ * * / * *
* init MasterNodeListener listener * init MasterNodeListener listener
* / * /
@ -107,8 +152,8 @@ public class ZookeeperNodeManager implements InitializingBean {
/ * * / * *
* master nodes from zookeeper * master nodes from zookeeper
* / * /
Set < String > m asterNodes= registryCenter . getMasterNodesDirectly ( ) ; Set < String > initM asterNodes= registryCenter . getMasterNodesDirectly ( ) ;
syncMasterNodes ( m asterNodes) ; syncMasterNodes ( initM asterNodes) ;
/ * * / * *
* worker group nodes from zookeeper * worker group nodes from zookeeper
@ -119,6 +164,43 @@ public class ZookeeperNodeManager implements InitializingBean {
} }
} }
/ * *
* zookeeper client
* /
@Component
static class ZKClient extends AbstractZKClient { }
/ * *
* worker node info and worker group db sync task
* /
class WorkerNodeInfoAndGroupDbSyncTask implements Runnable {
@Override
public void run ( ) {
// sync worker node info
Map < String , String > newWorkerNodeInfo = zkClient . getServerMaps ( ZKNodeType . WORKER , true ) ;
syncWorkerNodeInfo ( newWorkerNodeInfo ) ;
// sync worker group nodes from database
List < WorkerGroup > workerGroupList = workerGroupMapper . queryAllWorkerGroup ( ) ;
if ( CollectionUtils . isNotEmpty ( workerGroupList ) ) {
for ( WorkerGroup wg : workerGroupList ) {
String workerGroup = wg . getName ( ) ;
Set < String > nodes = new HashSet < > ( ) ;
String [ ] addrs = wg . getAddrList ( ) . split ( Constants . COMMA ) ;
for ( String addr : addrs ) {
if ( newWorkerNodeInfo . containsKey ( addr ) ) {
nodes . add ( addr ) ;
}
}
if ( ! nodes . isEmpty ( ) ) {
syncWorkerGroupNodes ( workerGroup , nodes ) ;
}
}
}
}
}
/ * * / * *
* worker group node listener * worker group node listener
* / * /
@ -141,8 +223,8 @@ public class ZookeeperNodeManager implements InitializingBean {
syncWorkerGroupNodes ( group , currentNodes ) ; syncWorkerGroupNodes ( group , currentNodes ) ;
alertDao . sendServerStopedAlert ( 1 , path , "WORKER" ) ; alertDao . sendServerStopedAlert ( 1 , path , "WORKER" ) ;
} }
} catch ( IllegalArgumentException ignor e) { } catch ( IllegalArgumentException ex ) {
logger . warn ( ignor e. getMessage ( ) ) ; logger . warn ( ex . getMessage ( ) ) ;
} catch ( Exception ex ) { } catch ( Exception ex ) {
logger . error ( "WorkerGroupListener capture data change and get data failed" , ex ) ; logger . error ( "WorkerGroupListener capture data change and get data failed" , ex ) ;
} }
@ -150,16 +232,14 @@ public class ZookeeperNodeManager implements InitializingBean {
} }
private String parseGroup ( String path ) { private String parseGroup ( String path ) {
String [ ] parts = path . split ( "\\ /" ) ; String [ ] parts = path . split ( "/" ) ;
if ( parts . length < 6 ) { if ( parts . length < 6 ) {
throw new IllegalArgumentException ( String . format ( "worker group path : %s is not valid, ignore" , path ) ) ; throw new IllegalArgumentException ( String . format ( "worker group path : %s is not valid, ignore" , path ) ) ;
} }
String group = parts [ parts . length - 2 ] ; return parts [ parts . length - 2 ] ;
return group ;
} }
} }
/ * * / * *
* master node listener * master node listener
* / * /
@ -244,7 +324,7 @@ public class ZookeeperNodeManager implements InitializingBean {
workerGroupLock . lock ( ) ; workerGroupLock . lock ( ) ;
try { try {
if ( StringUtils . isEmpty ( workerGroup ) ) { if ( StringUtils . isEmpty ( workerGroup ) ) {
workerGroup = DEFAULT_WORKER_GROUP ; workerGroup = Constants . DEFAULT_WORKER_GROUP ;
} }
workerGroup = workerGroup . toLowerCase ( ) ; workerGroup = workerGroup . toLowerCase ( ) ;
Set < String > nodes = workerGroupNodes . get ( workerGroup ) ; Set < String > nodes = workerGroupNodes . get ( workerGroup ) ;
@ -258,9 +338,48 @@ public class ZookeeperNodeManager implements InitializingBean {
} }
/ * * / * *
* close * get worker node info
* @return worker node info
* / * /
public void close ( ) { public Map < String , String > getWorkerNodeInfo ( ) {
return Collections . unmodifiableMap ( workerNodeInfo ) ;
}
/ * *
* get worker node info
* @param workerNode worker node
* @return worker node info
* /
public String getWorkerNodeInfo ( String workerNode ) {
workerNodeInfoLock . lock ( ) ;
try {
return workerNodeInfo . getOrDefault ( workerNode , null ) ;
} finally {
workerNodeInfoLock . unlock ( ) ;
}
}
/ * *
* sync worker node info
* @param newWorkerNodeInfo new worker node info
* /
private void syncWorkerNodeInfo ( Map < String , String > newWorkerNodeInfo ) {
workerNodeInfoLock . lock ( ) ;
try {
workerNodeInfo . clear ( ) ;
workerNodeInfo . putAll ( newWorkerNodeInfo ) ;
} finally {
workerNodeInfoLock . unlock ( ) ;
}
}
/ * *
* destroy
* /
@PreDestroy
public void destroy ( ) {
executorService . shutdownNow ( ) ;
registryCenter . close ( ) ; registryCenter . close ( ) ;
} }
} }
Shiwen Cheng
4 years ago
committed by
GitHub