/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.server.master.service; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_TASK_TYPE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.doNothing; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.runner.IWorkflowExecuteContext; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; import com.google.common.collect.Lists; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class FailoverServiceTest { private FailoverService failoverService; @Mock private MasterConfig masterConfig; @Mock private RegistryClient registryClient; @Mock private ProcessService processService; @Mock private TaskInstanceDao taskInstanceDao; @Mock private WorkflowExecuteThreadPool workflowExecuteThreadPool; @Mock private ProcessInstanceExecCacheManager cacheManager; @Mock private ProcessInstanceExecCacheManager processInstanceExecCacheManager; private static int masterPort = 5678; private static int workerPort = 1234; private String testMasterHost; private String testWorkerHost; private ProcessInstance processInstance; private TaskInstance masterTaskInstance; private TaskInstance workerTaskInstance; @BeforeEach public void before() throws Exception { ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); SpringApplicationContext springApplicationContext = new SpringApplicationContext(); springApplicationContext.setApplicationContext(applicationContext); given(masterConfig.getListenPort()).willReturn(masterPort); testMasterHost = NetUtils.getAddr(masterConfig.getListenPort()); given(masterConfig.getMasterAddress()).willReturn(testMasterHost); MasterFailoverService masterFailoverService = new MasterFailoverService(registryClient, masterConfig, processService, processInstanceExecCacheManager); WorkerFailoverService workerFailoverService = new WorkerFailoverService(registryClient, masterConfig, processService, workflowExecuteThreadPool, cacheManager, taskInstanceDao); failoverService = new FailoverService(masterFailoverService, workerFailoverService); String ip = testMasterHost.split(":")[0]; int port = Integer.parseInt(testMasterHost.split(":")[1]); Assertions.assertEquals(masterPort, port); testWorkerHost = ip + ":" + workerPort; given(registryClient.getLock(Mockito.anyString())).willReturn(true); given(registryClient.releaseLock(Mockito.anyString())).willReturn(true); processInstance = new ProcessInstance(); processInstance.setId(1); processInstance.setHost(testMasterHost); processInstance.setStartTime(new Date()); processInstance.setRestartTime(new Date()); processInstance.setHistoryCmd("xxx"); processInstance.setCommandType(CommandType.STOP); processInstance.setProcessDefinitionCode(123L); processInstance.setProcessDefinition(new ProcessDefinition()); masterTaskInstance = new TaskInstance(); masterTaskInstance.setId(1); masterTaskInstance.setStartTime(new Date()); masterTaskInstance.setHost(testMasterHost); masterTaskInstance.setTaskType(TASK_TYPE_SWITCH); workerTaskInstance = new TaskInstance(); workerTaskInstance.setId(2); workerTaskInstance.setStartTime(new Date()); workerTaskInstance.setHost(testWorkerHost); workerTaskInstance.setTaskType(COMMON_TASK_TYPE); given(processService.queryNeedFailoverProcessInstances(Mockito.anyString())) .willReturn(Arrays.asList(processInstance)); doNothing().when(processService).processNeedFailoverProcessInstances(Mockito.any(ProcessInstance.class)); given(taskInstanceDao.queryValidTaskListByWorkflowInstanceId(Mockito.anyInt(), Mockito.anyInt())) .willReturn(Lists.newArrayList(masterTaskInstance, workerTaskInstance)); Thread.sleep(1000); Server masterServer = new Server(); masterServer.setHost(ip); masterServer.setPort(masterPort); masterServer.setCreateTime(new Date()); Server workerServer = new Server(); workerServer.setHost(ip); workerServer.setPort(workerPort); workerServer.setCreateTime(new Date()); given(registryClient.getServerList(RegistryNodeType.WORKER)) .willReturn(new ArrayList<>(Arrays.asList(workerServer))); given(registryClient.getServerList(RegistryNodeType.MASTER)) .willReturn(new ArrayList<>(Arrays.asList(masterServer))); doNothing().when(workflowExecuteThreadPool).submitStateEvent(Mockito.any(StateEvent.class)); } @Test public void failoverMasterTest() { processInstance.setHost(Constants.NULL); masterTaskInstance.setState(TaskExecutionStatus.RUNNING_EXECUTION); failoverService.failoverServerWhenDown(testMasterHost, RegistryNodeType.MASTER); Assertions.assertNotEquals(masterTaskInstance.getState(), TaskExecutionStatus.NEED_FAULT_TOLERANCE); processInstance.setHost(testMasterHost); masterTaskInstance.setState(TaskExecutionStatus.SUCCESS); failoverService.failoverServerWhenDown(testMasterHost, RegistryNodeType.MASTER); Assertions.assertNotEquals(masterTaskInstance.getState(), TaskExecutionStatus.NEED_FAULT_TOLERANCE); processInstance.setHost(testMasterHost); masterTaskInstance.setState(TaskExecutionStatus.RUNNING_EXECUTION); failoverService.failoverServerWhenDown(testMasterHost, RegistryNodeType.MASTER); Assertions.assertEquals(masterTaskInstance.getState(), TaskExecutionStatus.RUNNING_EXECUTION); } @Test public void failoverWorkTest() { workerTaskInstance.setState(TaskExecutionStatus.RUNNING_EXECUTION); WorkflowExecuteRunnable workflowExecuteRunnable = Mockito.mock(WorkflowExecuteRunnable.class); Mockito.when(workflowExecuteRunnable.getAllTaskInstances()).thenReturn(Lists.newArrayList(workerTaskInstance)); IWorkflowExecuteContext workflowExecuteRunnableContext = Mockito.mock(IWorkflowExecuteContext.class); Mockito.when(workflowExecuteRunnable.getWorkflowExecuteContext()).thenReturn(workflowExecuteRunnableContext); Mockito.when(workflowExecuteRunnableContext.getWorkflowInstance()).thenReturn(processInstance); Mockito.when(cacheManager.getAll()).thenReturn(Lists.newArrayList(workflowExecuteRunnable)); Mockito.when(cacheManager.getByProcessInstanceId(Mockito.anyInt())).thenReturn(workflowExecuteRunnable); failoverService.failoverServerWhenDown(testWorkerHost, RegistryNodeType.WORKER); Assertions.assertEquals(TaskExecutionStatus.NEED_FAULT_TOLERANCE, workerTaskInstance.getState()); } }