@ -43,6 +43,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext ; import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext ;
import org.apache.dolphinscheduler.service.registry.RegistryClient ; import org.apache.dolphinscheduler.service.registry.RegistryClient ;
import org.apache.commons.collections4.CollectionUtils ;
import org.apache.commons.lang.StringUtils ; import org.apache.commons.lang.StringUtils ;
import java.util.Collections ; import java.util.Collections ;
@ -127,16 +128,16 @@ public class MasterRegistryClient {
ThreadUtils . sleep ( SLEEP_TIME_MILLIS ) ; ThreadUtils . sleep ( SLEEP_TIME_MILLIS ) ;
} }
// self tolerant
if ( registryClient . getActiveMasterNum ( ) = = 1 ) {
removeNodePath ( null , NodeType . MASTER , true ) ;
removeNodePath ( null , NodeType . WORKER , true ) ;
}
registryClient . subscribe ( REGISTRY_DOLPHINSCHEDULER_NODE , new MasterRegistryDataListener ( ) ) ; registryClient . subscribe ( REGISTRY_DOLPHINSCHEDULER_NODE , new MasterRegistryDataListener ( ) ) ;
} catch ( Exception e ) { } catch ( Exception e ) {
logger . error ( "master start up exception" , e ) ; logger . error ( "master start up exception" , e ) ;
this . registryClient . getStoppable ( ) . stop ( "master start up exception" ) ;
} finally { } finally {
try {
registryClient . releaseLock ( nodeLock ) ; registryClient . releaseLock ( nodeLock ) ;
} catch ( Exception e ) {
logger . error ( "release lock error" , e ) ;
}
} }
} }
@ -150,18 +151,57 @@ public class MasterRegistryClient {
} }
/ * * / * *
* remove zookeep ernode path * remove mast ernode path
* *
* @param path zookeeper node path * @param path node path
* @param nodeType zookeeper node type * @param nodeType node type
* @param failover is failover * @param failover is failover
* / * /
public void removeNodePath ( String path , NodeType nodeType , boolean failover ) { public void removeMaster NodePath ( String path , NodeType nodeType , boolean failover ) {
logger . info ( "{} node deleted : {}" , nodeType , path ) ; logger . info ( "{} node deleted : {}" , nodeType , path ) ;
String failoverPath = getFailoverLockPath ( nodeType ) ;
if ( StringUtils . isEmpty ( path ) ) {
logger . error ( "server down error: empty path: {}, nodeType:{}" , path , nodeType ) ;
return ;
}
String serverHost = registryClient . getHostByEventDataPath ( path ) ;
if ( StringUtils . isEmpty ( serverHost ) ) {
logger . error ( "server down error: unknown path: {}, nodeType:{}" , path , nodeType ) ;
return ;
}
String failoverPath = getFailoverLockPath ( nodeType , serverHost ) ;
try { try {
registryClient . getLock ( failoverPath ) ; registryClient . getLock ( failoverPath ) ;
if ( ! registryClient . exists ( path ) ) {
logger . info ( "path: {} not exists" , path ) ;
// handle dead server
registryClient . handleDeadServer ( Collections . singleton ( path ) , nodeType , Constants . ADD_OP ) ;
}
//failover server
if ( failover ) {
failoverServerWhenDown ( serverHost , nodeType ) ;
}
} catch ( Exception e ) {
logger . error ( "{} server failover failed, host:{}" , nodeType , serverHost , e ) ;
} finally {
registryClient . releaseLock ( failoverPath ) ;
}
}
/ * *
* remove worker node path
*
* @param path node path
* @param nodeType node type
* @param failover is failover
* /
public void removeWorkerNodePath ( String path , NodeType nodeType , boolean failover ) {
logger . info ( "{} node deleted : {}" , nodeType , path ) ;
try {
String serverHost = null ; String serverHost = null ;
if ( ! StringUtils . isEmpty ( path ) ) { if ( ! StringUtils . isEmpty ( path ) ) {
serverHost = registryClient . getHostByEventDataPath ( path ) ; serverHost = registryClient . getHostByEventDataPath ( path ) ;
@ -169,18 +209,18 @@ public class MasterRegistryClient {
logger . error ( "server down error: unknown path: {}" , path ) ; logger . error ( "server down error: unknown path: {}" , path ) ;
return ; return ;
} }
if ( ! registryClient . exists ( path ) ) {
logger . info ( "path: {} not exists" , path ) ;
// handle dead server
// handle dead server
registryClient . handleDeadServer ( Collections . singleton ( path ) , nodeType , Constants . ADD_OP ) ; registryClient . handleDeadServer ( Collections . singleton ( path ) , nodeType , Constants . ADD_OP ) ;
} }
}
//failover server
//failover server
if ( failover ) { if ( failover ) {
failoverServerWhenDown ( serverHost , nodeType ) ; failoverServerWhenDown ( serverHost , nodeType ) ;
} }
} catch ( Exception e ) { } catch ( Exception e ) {
logger . error ( "{} server failover failed." , nodeType ) ; logger . error ( "{} server failover failed" , nodeType , e ) ;
logger . error ( "failover exception " , e ) ;
} finally {
registryClient . releaseLock ( failoverPath ) ;
} }
} }
@ -209,12 +249,12 @@ public class MasterRegistryClient {
* @param nodeType zookeeper node type * @param nodeType zookeeper node type
* @return fail over lock path * @return fail over lock path
* / * /
private String getFailoverLockPath ( NodeType nodeType ) { public String getFailoverLockPath ( NodeType nodeType , String host ) {
switch ( nodeType ) { switch ( nodeType ) {
case MASTER : case MASTER :
return Constants . REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS ; return Constants . REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS + "/" + host ;
case WORKER : case WORKER :
return Constants . REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS ; return Constants . REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS + "/" + host ;
default : default :
return "" ; return "" ;
} }
@ -226,7 +266,11 @@ public class MasterRegistryClient {
* @param taskInstance task instance * @param taskInstance task instance
* @return true if task instance need fail over * @return true if task instance need fail over
* / * /
private boolean checkTaskInstanceNeedFailover ( TaskInstance taskInstance ) { private boolean checkTaskInstanceNeedFailover ( List < Server > workerServers , TaskInstance taskInstance ) {
// first submit: host is null
// dispatch succeed: host is not null && submit_time is null
// ACK || RESULT from worker: host is not null && start_time is not null
boolean taskNeedFailover = true ; boolean taskNeedFailover = true ;
@ -234,14 +278,15 @@ public class MasterRegistryClient {
if ( taskInstance . getHost ( ) = = null ) { if ( taskInstance . getHost ( ) = = null ) {
return false ; return false ;
} }
// host is not null and submit time is null, master will retry
// if the worker node exists in zookeeper, we must check the task starts after the worker
if ( taskInstance . getSubmitTime ( ) = = null ) {
if ( registryClient . checkNodeExists ( taskInstance . getHost ( ) , NodeType . WORKER ) ) { return false ;
}
//if task start after worker starts, there is no need to failover the task.
//if task start after worker starts, there is no need to failover the task.
if ( checkTaskAfterWorkerStart ( taskInstance ) ) { if ( checkTaskAfterWorkerStart ( workerServers , taskInstance ) ) {
taskNeedFailover = false ; taskNeedFailover = false ;
} }
}
return taskNeedFailover ; return taskNeedFailover ;
} }
@ -269,6 +314,54 @@ public class MasterRegistryClient {
return false ; return false ;
} }
/ * *
* check task start after the worker server starts .
*
* @param taskInstance task instance
* @return true if task instance start time after worker server start date
* /
private boolean checkTaskAfterWorkerStart ( List < Server > workerServers , TaskInstance taskInstance ) {
if ( StringUtils . isEmpty ( taskInstance . getHost ( ) ) ) {
return false ;
}
Date taskTime = taskInstance . getStartTime ( ) = = null ? taskInstance . getSubmitTime ( ) : taskInstance . getStartTime ( ) ;
Date workerServerStartDate = getServerStartupTime ( workerServers , taskInstance . getHost ( ) ) ;
if ( workerServerStartDate ! = null ) {
return taskTime . after ( workerServerStartDate ) ;
}
return false ;
}
/ * *
* get server startup time
* /
private Date getServerStartupTime ( List < Server > servers , String host ) {
if ( CollectionUtils . isEmpty ( servers ) ) {
return null ;
}
Date serverStartupTime = null ;
for ( Server server : servers ) {
if ( host . equals ( server . getHost ( ) + Constants . COLON + server . getPort ( ) ) ) {
serverStartupTime = server . getCreateTime ( ) ;
break ;
}
}
return serverStartupTime ;
}
/ * *
* get server startup time
* /
private Date getServerStartupTime ( NodeType nodeType , String host ) {
if ( StringUtils . isEmpty ( host ) ) {
return null ;
}
List < Server > servers = registryClient . getServerList ( nodeType ) ;
return getServerStartupTime ( servers , host ) ;
}
/ * * / * *
* failover worker tasks * failover worker tasks
* < p > * < p >
@ -279,10 +372,13 @@ public class MasterRegistryClient {
* @param workerHost worker host * @param workerHost worker host
* / * /
private void failoverWorker ( String workerHost ) { private void failoverWorker ( String workerHost ) {
if ( StringUtils . isEmpty ( workerHost ) ) { if ( StringUtils . isEmpty ( workerHost ) ) {
return ; return ;
} }
List < Server > workerServers = registryClient . getServerList ( NodeType . WORKER ) ;
long startTime = System . currentTimeMillis ( ) ; long startTime = System . currentTimeMillis ( ) ;
List < TaskInstance > needFailoverTaskInstanceList = processService . queryNeedFailoverTaskInstances ( workerHost ) ; List < TaskInstance > needFailoverTaskInstanceList = processService . queryNeedFailoverTaskInstances ( workerHost ) ;
Map < Integer , ProcessInstance > processInstanceCacheMap = new HashMap < > ( ) ; Map < Integer , ProcessInstance > processInstanceCacheMap = new HashMap < > ( ) ;
@ -300,12 +396,18 @@ public class MasterRegistryClient {
processInstanceCacheMap . put ( processInstance . getId ( ) , processInstance ) ; processInstanceCacheMap . put ( processInstance . getId ( ) , processInstance ) ;
} }
if ( ! checkTaskInstanceNeedFailover ( workerServers , taskInstance ) ) {
continue ;
}
// only failover the task owned myself if worker down.
// only failover the task owned myself if worker down.
if ( processInstance . getHost ( ) . equalsIgnoreCase ( getLocalAddress ( ) ) ) { if ( ! processInstance . getHost ( ) . equalsIgnoreCase ( getLocalAddress ( ) ) ) {
continue ;
}
logger . info ( "failover task instance id: {}, process instance id: {}" , taskInstance . getId ( ) , taskInstance . getProcessInstanceId ( ) ) ; logger . info ( "failover task instance id: {}, process instance id: {}" , taskInstance . getId ( ) , taskInstance . getProcessInstanceId ( ) ) ;
failoverTaskInstance ( processInstance , taskInstance ) ; failoverTaskInstance ( processInstance , taskInstance ) ;
} }
}
logger . info ( "end worker[{}] failover, useTime:{}ms" , workerHost , System . currentTimeMillis ( ) - startTime ) ; logger . info ( "end worker[{}] failover, useTime:{}ms" , workerHost , System . currentTimeMillis ( ) - startTime ) ;
} }
@ -316,11 +418,15 @@ public class MasterRegistryClient {
* *
* @param masterHost master host * @param masterHost master host
* / * /
private void failoverMaster ( String masterHost ) { public void failoverMaster ( String masterHost ) {
if ( StringUtils . isEmpty ( masterHost ) ) { if ( StringUtils . isEmpty ( masterHost ) ) {
return ; return ;
} }
Date serverStartupTime = getServerStartupTime ( NodeType . MASTER , masterHost ) ;
List < Server > workerServers = registryClient . getServerList ( NodeType . WORKER ) ;
long startTime = System . currentTimeMillis ( ) ; long startTime = System . currentTimeMillis ( ) ;
List < ProcessInstance > needFailoverProcessInstanceList = processService . queryNeedFailoverProcessInstances ( masterHost ) ; List < ProcessInstance > needFailoverProcessInstanceList = processService . queryNeedFailoverProcessInstances ( masterHost ) ;
logger . info ( "start master[{}] failover, process list size:{}" , masterHost , needFailoverProcessInstanceList . size ( ) ) ; logger . info ( "start master[{}] failover, process list size:{}" , masterHost , needFailoverProcessInstanceList . size ( ) ) ;
@ -330,6 +436,11 @@ public class MasterRegistryClient {
continue ; continue ;
} }
if ( serverStartupTime ! = null & & processInstance . getRestartTime ( ) ! = null
& & processInstance . getRestartTime ( ) . after ( serverStartupTime ) ) {
continue ;
}
logger . info ( "failover process instance id: {}" , processInstance . getId ( ) ) ; logger . info ( "failover process instance id: {}" , processInstance . getId ( ) ) ;
List < TaskInstance > validTaskInstanceList = processService . findValidTaskListByProcessId ( processInstance . getId ( ) ) ; List < TaskInstance > validTaskInstanceList = processService . findValidTaskListByProcessId ( processInstance . getId ( ) ) ;
@ -337,6 +448,12 @@ public class MasterRegistryClient {
if ( Constants . NULL . equals ( taskInstance . getHost ( ) ) ) { if ( Constants . NULL . equals ( taskInstance . getHost ( ) ) ) {
continue ; continue ;
} }
if ( taskInstance . getState ( ) . typeIsFinished ( ) ) {
continue ;
}
if ( ! checkTaskInstanceNeedFailover ( workerServers , taskInstance ) ) {
continue ;
}
logger . info ( "failover task instance id: {}, process instance id: {}" , taskInstance . getId ( ) , taskInstance . getProcessInstanceId ( ) ) ; logger . info ( "failover task instance id: {}, process instance id: {}" , taskInstance . getId ( ) , taskInstance . getProcessInstanceId ( ) ) ;
failoverTaskInstance ( processInstance , taskInstance ) ; failoverTaskInstance ( processInstance , taskInstance ) ;
} }
@ -347,6 +464,13 @@ public class MasterRegistryClient {
logger . info ( "master[{}] failover end, useTime:{}ms" , masterHost , System . currentTimeMillis ( ) - startTime ) ; logger . info ( "master[{}] failover end, useTime:{}ms" , masterHost , System . currentTimeMillis ( ) - startTime ) ;
} }
/ * *
* failover task instance
* < p >
* 1 . kill yarn job if there are yarn jobs in tasks .
* 2 . change task state from running to need failover .
* 3 . try to notify local master
* /
private void failoverTaskInstance ( ProcessInstance processInstance , TaskInstance taskInstance ) { private void failoverTaskInstance ( ProcessInstance processInstance , TaskInstance taskInstance ) {
if ( taskInstance = = null ) { if ( taskInstance = = null ) {
logger . error ( "failover task instance error, taskInstance is null" ) ; logger . error ( "failover task instance error, taskInstance is null" ) ;
@ -359,24 +483,23 @@ public class MasterRegistryClient {
return ; return ;
} }
if ( ! checkTaskInstanceNeedFailover ( taskInstance ) ) {
return ;
}
taskInstance . setProcessInstance ( processInstance ) ; taskInstance . setProcessInstance ( processInstance ) ;
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder . get ( ) TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder . get ( )
. buildTaskInstanceRelatedInfo ( taskInstance ) . buildTaskInstanceRelatedInfo ( taskInstance )
. buildProcessInstanceRelatedInfo ( processInstance ) . buildProcessInstanceRelatedInfo ( processInstance )
. create ( ) ; . create ( ) ;
if ( masterConfig . getMasterKillYarnJobWhenHandleFailOver ( ) ) {
// only kill yarn job if exists , the local thread has exited
// only kill yarn job if exists , the local thread has exited
ProcessUtils . killYarnJob ( taskExecutionContext ) ; ProcessUtils . killYarnJob ( taskExecutionContext ) ;
}
taskInstance . setState ( ExecutionStatus . NEED_FAULT_TOLERANCE ) ; taskInstance . setState ( ExecutionStatus . NEED_FAULT_TOLERANCE ) ;
processService . saveTaskInstance ( taskInstance ) ; processService . saveTaskInstance ( taskInstance ) ;
WorkflowExecuteThread workflowExecuteThreadNotify = processInstanceExecMaps . get ( processInstance . getId ( ) ) ; WorkflowExecuteThread workflowExecuteThreadNotify = processInstanceExecMaps . get ( processInstance . getId ( ) ) ;
if ( workflowExecuteThreadNotify = = null ) { if ( workflowExecuteThreadNotify = = null ) {
logger . info ( "workflowExecuteThreadNotify is null, just return, task id:{},process id:{}" , taskInstance . getId ( ) , processInstance . getId ( ) ) ;
return ; return ;
} }
StateEvent stateEvent = new StateEvent ( ) ; StateEvent stateEvent = new StateEvent ( ) ;
zwZjut
3 years ago
committed by
GitHub