Browse Source
- add FailoverService.java - move failover method from MasterRegistryClient to FailoverService - move failover code from FailoverExecuteThread to FailoverService This closes #92633.0.0/version-upgrade
worry
3 years ago
committed by
GitHub
4 changed files with 539 additions and 335 deletions
@ -0,0 +1,369 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.service; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.NodeType; |
||||
import org.apache.dolphinscheduler.common.enums.StateEvent; |
||||
import org.apache.dolphinscheduler.common.enums.StateEventType; |
||||
import org.apache.dolphinscheduler.common.model.Server; |
||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.server.utils.ProcessUtils; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.apache.dolphinscheduler.service.registry.RegistryClient; |
||||
|
||||
import org.apache.commons.collections4.CollectionUtils; |
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.Date; |
||||
import java.util.HashMap; |
||||
import java.util.Iterator; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* failover service |
||||
*/ |
||||
@Component |
||||
public class FailoverService { |
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FailoverService.class); |
||||
private final RegistryClient registryClient; |
||||
private final MasterConfig masterConfig; |
||||
private final ProcessService processService; |
||||
private final WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
public FailoverService(RegistryClient registryClient, MasterConfig masterConfig, ProcessService processService, |
||||
WorkflowExecuteThreadPool workflowExecuteThreadPool) { |
||||
this.registryClient = registryClient; |
||||
this.masterConfig = masterConfig; |
||||
this.processService = processService; |
||||
this.workflowExecuteThreadPool = workflowExecuteThreadPool; |
||||
} |
||||
|
||||
/** |
||||
* check master failover |
||||
*/ |
||||
public void checkMasterFailover() { |
||||
List<String> hosts = getNeedFailoverMasterServers(); |
||||
if (CollectionUtils.isEmpty(hosts)) { |
||||
return; |
||||
} |
||||
LOGGER.info("need failover hosts:{}", hosts); |
||||
|
||||
for (String host : hosts) { |
||||
failoverMasterWithLock(host); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* failover server when server down |
||||
* |
||||
* @param serverHost server host |
||||
* @param nodeType node type |
||||
*/ |
||||
public void failoverServerWhenDown(String serverHost, NodeType nodeType) { |
||||
switch (nodeType) { |
||||
case MASTER: |
||||
failoverMasterWithLock(serverHost); |
||||
break; |
||||
case WORKER: |
||||
failoverWorker(serverHost); |
||||
break; |
||||
default: |
||||
break; |
||||
} |
||||
} |
||||
|
||||
private void failoverMasterWithLock(String masterHost) { |
||||
String failoverPath = getFailoverLockPath(NodeType.MASTER, masterHost); |
||||
try { |
||||
registryClient.getLock(failoverPath); |
||||
this.failoverMaster(masterHost); |
||||
} catch (Exception e) { |
||||
LOGGER.error("{} server failover failed, host:{}", NodeType.MASTER, masterHost, e); |
||||
} finally { |
||||
registryClient.releaseLock(failoverPath); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* failover master |
||||
* <p> |
||||
* failover process instance and associated task instance |
||||
* |
||||
* @param masterHost master host |
||||
*/ |
||||
private void failoverMaster(String masterHost) { |
||||
if (StringUtils.isEmpty(masterHost)) { |
||||
return; |
||||
} |
||||
Date serverStartupTime = getServerStartupTime(NodeType.MASTER, masterHost); |
||||
long startTime = System.currentTimeMillis(); |
||||
List<ProcessInstance> needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost); |
||||
LOGGER.info("start master[{}] failover, process list size:{}", masterHost, needFailoverProcessInstanceList.size()); |
||||
List<Server> workerServers = registryClient.getServerList(NodeType.WORKER); |
||||
for (ProcessInstance processInstance : needFailoverProcessInstanceList) { |
||||
if (Constants.NULL.equals(processInstance.getHost())) { |
||||
continue; |
||||
} |
||||
|
||||
List<TaskInstance> validTaskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); |
||||
for (TaskInstance taskInstance : validTaskInstanceList) { |
||||
LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId()); |
||||
failoverTaskInstance(processInstance, taskInstance, workerServers); |
||||
} |
||||
|
||||
if (serverStartupTime != null && processInstance.getRestartTime() != null |
||||
&& processInstance.getRestartTime().after(serverStartupTime)) { |
||||
continue; |
||||
} |
||||
|
||||
LOGGER.info("failover process instance id: {}", processInstance.getId()); |
||||
//updateProcessInstance host is null and insert into command
|
||||
processInstance.setHost(Constants.NULL); |
||||
processService.processNeedFailoverProcessInstances(processInstance); |
||||
} |
||||
|
||||
LOGGER.info("master[{}] failover end, useTime:{}ms", masterHost, System.currentTimeMillis() - startTime); |
||||
} |
||||
|
||||
/** |
||||
* failover worker tasks |
||||
* <p> |
||||
* 1. kill yarn job if there are yarn jobs in tasks. |
||||
* 2. change task state from running to need failover. |
||||
* 3. failover all tasks when workerHost is null |
||||
* |
||||
* @param workerHost worker host |
||||
*/ |
||||
private void failoverWorker(String workerHost) { |
||||
if (StringUtils.isEmpty(workerHost)) { |
||||
return; |
||||
} |
||||
|
||||
long startTime = System.currentTimeMillis(); |
||||
List<TaskInstance> needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost); |
||||
Map<Integer, ProcessInstance> processInstanceCacheMap = new HashMap<>(); |
||||
LOGGER.info("start worker[{}] failover, task list size:{}", workerHost, needFailoverTaskInstanceList.size()); |
||||
List<Server> workerServers = registryClient.getServerList(NodeType.WORKER); |
||||
for (TaskInstance taskInstance : needFailoverTaskInstanceList) { |
||||
ProcessInstance processInstance = processInstanceCacheMap.get(taskInstance.getProcessInstanceId()); |
||||
if (processInstance == null) { |
||||
processInstance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); |
||||
if (processInstance == null) { |
||||
LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null", |
||||
taskInstance.getProcessInstanceId(), taskInstance.getId()); |
||||
continue; |
||||
} |
||||
processInstanceCacheMap.put(processInstance.getId(), processInstance); |
||||
} |
||||
|
||||
// only failover the task owned myself if worker down.
|
||||
if (!processInstance.getHost().equalsIgnoreCase(getLocalAddress())) { |
||||
continue; |
||||
} |
||||
|
||||
LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId()); |
||||
failoverTaskInstance(processInstance, taskInstance, workerServers); |
||||
} |
||||
LOGGER.info("end worker[{}] failover, useTime:{}ms", workerHost, System.currentTimeMillis() - startTime); |
||||
} |
||||
|
||||
/** |
||||
* failover task instance |
||||
* <p> |
||||
* 1. kill yarn job if there are yarn jobs in tasks. |
||||
* 2. change task state from running to need failover. |
||||
* 3. try to notify local master |
||||
*/ |
||||
private void failoverTaskInstance(ProcessInstance processInstance, TaskInstance taskInstance, List<Server> workerServers) { |
||||
if (processInstance == null) { |
||||
LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null", |
||||
taskInstance.getProcessInstanceId(), taskInstance.getId()); |
||||
return; |
||||
} |
||||
if (!checkTaskInstanceNeedFailover(workerServers, taskInstance)) { |
||||
return; |
||||
} |
||||
|
||||
taskInstance.setProcessInstance(processInstance); |
||||
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() |
||||
.buildTaskInstanceRelatedInfo(taskInstance) |
||||
.buildProcessInstanceRelatedInfo(processInstance) |
||||
.create(); |
||||
|
||||
if (masterConfig.isKillYarnJobWhenTaskFailover()) { |
||||
// only kill yarn job if exists , the local thread has exited
|
||||
ProcessUtils.killYarnJob(taskExecutionContext); |
||||
} |
||||
|
||||
taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); |
||||
processService.saveTaskInstance(taskInstance); |
||||
|
||||
StateEvent stateEvent = new StateEvent(); |
||||
stateEvent.setTaskInstanceId(taskInstance.getId()); |
||||
stateEvent.setType(StateEventType.TASK_STATE_CHANGE); |
||||
stateEvent.setProcessInstanceId(processInstance.getId()); |
||||
stateEvent.setExecutionStatus(taskInstance.getState()); |
||||
workflowExecuteThreadPool.submitStateEvent(stateEvent); |
||||
} |
||||
|
||||
/** |
||||
* get need failover master servers |
||||
* |
||||
* @return need failover master servers |
||||
*/ |
||||
private List<String> getNeedFailoverMasterServers() { |
||||
// failover myself && failover dead masters
|
||||
List<String> hosts = processService.queryNeedFailoverProcessInstanceHost(); |
||||
|
||||
Iterator<String> iterator = hosts.iterator(); |
||||
while (iterator.hasNext()) { |
||||
String host = iterator.next(); |
||||
if (registryClient.checkNodeExists(host, NodeType.MASTER)) { |
||||
if (!host.equals(getLocalAddress())) { |
||||
iterator.remove(); |
||||
} |
||||
} |
||||
} |
||||
return hosts; |
||||
} |
||||
|
||||
/** |
||||
* task needs failover if task start before worker starts |
||||
* |
||||
* @param workerServers worker servers |
||||
* @param taskInstance task instance |
||||
* @return true if task instance need fail over |
||||
*/ |
||||
private boolean checkTaskInstanceNeedFailover(List<Server> workerServers, TaskInstance taskInstance) { |
||||
|
||||
boolean taskNeedFailover = true; |
||||
|
||||
if (taskInstance == null) { |
||||
LOGGER.error("failover task instance error, taskInstance is null"); |
||||
return false; |
||||
} |
||||
|
||||
if (Constants.NULL.equals(taskInstance.getHost())) { |
||||
return false; |
||||
} |
||||
|
||||
if (taskInstance.getState() != null && taskInstance.getState().typeIsFinished()) { |
||||
return false; |
||||
} |
||||
|
||||
|
||||
//now no host will execute this task instance,so no need to failover the task
|
||||
if (taskInstance.getHost() == null) { |
||||
return false; |
||||
} |
||||
|
||||
//if task start after worker starts, there is no need to failover the task.
|
||||
if (checkTaskAfterWorkerStart(workerServers, taskInstance)) { |
||||
taskNeedFailover = false; |
||||
} |
||||
|
||||
return taskNeedFailover; |
||||
} |
||||
|
||||
/** |
||||
* check task start after the worker server starts. |
||||
* |
||||
* @param taskInstance task instance |
||||
* @return true if task instance start time after worker server start date |
||||
*/ |
||||
private boolean checkTaskAfterWorkerStart(List<Server> workerServers, TaskInstance taskInstance) { |
||||
if (StringUtils.isEmpty(taskInstance.getHost())) { |
||||
return false; |
||||
} |
||||
Date workerServerStartDate = getServerStartupTime(workerServers, taskInstance.getHost()); |
||||
if (workerServerStartDate != null) { |
||||
if (taskInstance.getStartTime() == null) { |
||||
return taskInstance.getSubmitTime().after(workerServerStartDate); |
||||
} else { |
||||
return taskInstance.getStartTime().after(workerServerStartDate); |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* get failover lock path |
||||
* |
||||
* @param nodeType zookeeper node type |
||||
* @return fail over lock path |
||||
*/ |
||||
private String getFailoverLockPath(NodeType nodeType, String host) { |
||||
switch (nodeType) { |
||||
case MASTER: |
||||
return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS + "/" + host; |
||||
case WORKER: |
||||
return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS + "/" + host; |
||||
default: |
||||
return ""; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get server startup time |
||||
*/ |
||||
private Date getServerStartupTime(NodeType nodeType, String host) { |
||||
if (StringUtils.isEmpty(host)) { |
||||
return null; |
||||
} |
||||
List<Server> servers = registryClient.getServerList(nodeType); |
||||
return getServerStartupTime(servers, host); |
||||
} |
||||
|
||||
/** |
||||
* get server startup time |
||||
*/ |
||||
private Date getServerStartupTime(List<Server> servers, String host) { |
||||
if (CollectionUtils.isEmpty(servers)) { |
||||
return null; |
||||
} |
||||
Date serverStartupTime = null; |
||||
for (Server server : servers) { |
||||
if (host.equals(server.getHost() + Constants.COLON + server.getPort())) { |
||||
serverStartupTime = server.getCreateTime(); |
||||
break; |
||||
} |
||||
} |
||||
return serverStartupTime; |
||||
} |
||||
|
||||
/** |
||||
* get local address |
||||
*/ |
||||
String getLocalAddress() { |
||||
return NetUtils.getAddr(masterConfig.getListenPort()); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,157 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.service; |
||||
|
||||
import static org.mockito.BDDMockito.given; |
||||
import static org.mockito.Mockito.doNothing; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.common.enums.NodeType; |
||||
import org.apache.dolphinscheduler.common.enums.StateEvent; |
||||
import org.apache.dolphinscheduler.common.model.Server; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.apache.dolphinscheduler.service.registry.RegistryClient; |
||||
|
||||
import java.util.Arrays; |
||||
import java.util.Date; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.powermock.core.classloader.annotations.PowerMockIgnore; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
import org.springframework.test.util.ReflectionTestUtils; |
||||
|
||||
import com.google.common.collect.Lists; |
||||
|
||||
/** |
||||
* MasterRegistryClientTest |
||||
*/ |
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({RegistryClient.class}) |
||||
@PowerMockIgnore({"javax.management.*"}) |
||||
public class FailoverServiceTest { |
||||
@InjectMocks |
||||
private FailoverService failoverService; |
||||
|
||||
@Mock |
||||
private MasterConfig masterConfig; |
||||
|
||||
@Mock |
||||
private RegistryClient registryClient; |
||||
|
||||
@Mock |
||||
private ProcessService processService; |
||||
|
||||
@Mock |
||||
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
private String testHost; |
||||
private ProcessInstance processInstance; |
||||
private TaskInstance taskInstance; |
||||
|
||||
@Before |
||||
public void before() throws Exception { |
||||
given(masterConfig.getListenPort()).willReturn(8080); |
||||
|
||||
testHost = failoverService.getLocalAddress(); |
||||
String ip = testHost.split(":")[0]; |
||||
int port = Integer.valueOf(testHost.split(":")[1]); |
||||
Assert.assertEquals(8080, port); |
||||
|
||||
given(registryClient.getLock(Mockito.anyString())).willReturn(true); |
||||
given(registryClient.releaseLock(Mockito.anyString())).willReturn(true); |
||||
given(registryClient.getHostByEventDataPath(Mockito.anyString())).willReturn(testHost); |
||||
given(registryClient.getStoppable()).willReturn(cause -> { |
||||
}); |
||||
given(registryClient.checkNodeExists(Mockito.anyString(), Mockito.any())).willReturn(true); |
||||
doNothing().when(registryClient).handleDeadServer(Mockito.anySet(), Mockito.any(NodeType.class), Mockito.anyString()); |
||||
|
||||
processInstance = new ProcessInstance(); |
||||
processInstance.setId(1); |
||||
processInstance.setHost(testHost); |
||||
processInstance.setRestartTime(new Date()); |
||||
processInstance.setHistoryCmd("xxx"); |
||||
processInstance.setCommandType(CommandType.STOP); |
||||
|
||||
taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setStartTime(new Date()); |
||||
taskInstance.setHost(testHost); |
||||
|
||||
given(processService.queryNeedFailoverTaskInstances(Mockito.anyString())).willReturn(Arrays.asList(taskInstance)); |
||||
given(processService.queryNeedFailoverProcessInstanceHost()).willReturn(Lists.newArrayList(testHost)); |
||||
given(processService.queryNeedFailoverProcessInstances(Mockito.anyString())).willReturn(Arrays.asList(processInstance)); |
||||
doNothing().when(processService).processNeedFailoverProcessInstances(Mockito.any(ProcessInstance.class)); |
||||
given(processService.findValidTaskListByProcessId(Mockito.anyInt())).willReturn(Lists.newArrayList(taskInstance)); |
||||
given(processService.findProcessInstanceDetailById(Mockito.anyInt())).willReturn(processInstance); |
||||
|
||||
Thread.sleep(1000); |
||||
Server server = new Server(); |
||||
server.setHost(ip); |
||||
server.setPort(port); |
||||
server.setCreateTime(new Date()); |
||||
given(registryClient.getServerList(NodeType.WORKER)).willReturn(Arrays.asList(server)); |
||||
given(registryClient.getServerList(NodeType.MASTER)).willReturn(Arrays.asList(server)); |
||||
ReflectionTestUtils.setField(failoverService, "registryClient", registryClient); |
||||
|
||||
doNothing().when(workflowExecuteThreadPool).submitStateEvent(Mockito.any(StateEvent.class)); |
||||
} |
||||
|
||||
@Test |
||||
public void checkMasterFailoverTest() { |
||||
failoverService.checkMasterFailover(); |
||||
} |
||||
|
||||
@Test |
||||
public void failoverMasterTest() { |
||||
processInstance.setHost(Constants.NULL); |
||||
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); |
||||
failoverService.failoverServerWhenDown(testHost, NodeType.MASTER); |
||||
Assert.assertNotEquals(taskInstance.getState(), ExecutionStatus.NEED_FAULT_TOLERANCE); |
||||
|
||||
processInstance.setHost(testHost); |
||||
taskInstance.setState(ExecutionStatus.SUCCESS); |
||||
failoverService.failoverServerWhenDown(testHost, NodeType.MASTER); |
||||
Assert.assertNotEquals(taskInstance.getState(), ExecutionStatus.NEED_FAULT_TOLERANCE); |
||||
Assert.assertEquals(Constants.NULL, processInstance.getHost()); |
||||
|
||||
processInstance.setHost(testHost); |
||||
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); |
||||
failoverService.failoverServerWhenDown(testHost, NodeType.MASTER); |
||||
Assert.assertEquals(taskInstance.getState(), ExecutionStatus.NEED_FAULT_TOLERANCE); |
||||
Assert.assertEquals(Constants.NULL, processInstance.getHost()); |
||||
} |
||||
|
||||
@Test |
||||
public void failoverWorkTest() { |
||||
failoverService.failoverServerWhenDown(testHost, NodeType.WORKER); |
||||
Assert.assertEquals(taskInstance.getState(), ExecutionStatus.NEED_FAULT_TOLERANCE); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue