wuxiaofei
4 years ago
64 changed files with 1459 additions and 326 deletions
@ -0,0 +1,143 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.Event; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||||
|
import org.apache.dolphinscheduler.common.thread.ThreadUtils; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
|
||||||
|
import java.util.concurrent.DelayQueue; |
||||||
|
import java.util.concurrent.ExecutorService; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
/** |
||||||
|
* Manage tasks |
||||||
|
*/ |
||||||
|
@Component |
||||||
|
public class WorkerManagerThread implements Runnable { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(WorkerManagerThread.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* task queue |
||||||
|
*/ |
||||||
|
private final DelayQueue<TaskExecuteThread> workerExecuteQueue = new DelayQueue<>(); |
||||||
|
|
||||||
|
/** |
||||||
|
* worker config |
||||||
|
*/ |
||||||
|
private final WorkerConfig workerConfig; |
||||||
|
|
||||||
|
/** |
||||||
|
* thread executor service |
||||||
|
*/ |
||||||
|
private final ExecutorService workerExecService; |
||||||
|
|
||||||
|
/** |
||||||
|
* taskExecutionContextCacheManager |
||||||
|
*/ |
||||||
|
private TaskExecutionContextCacheManager taskExecutionContextCacheManager; |
||||||
|
|
||||||
|
/** |
||||||
|
* task callback service |
||||||
|
*/ |
||||||
|
private final TaskCallbackService taskCallbackService; |
||||||
|
|
||||||
|
public WorkerManagerThread() { |
||||||
|
this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class); |
||||||
|
this.taskExecutionContextCacheManager = SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class); |
||||||
|
this.workerExecService = ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", this.workerConfig.getWorkerExecThreads()); |
||||||
|
this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get queue size |
||||||
|
* |
||||||
|
* @return queue size |
||||||
|
*/ |
||||||
|
public int getQueueSize() { |
||||||
|
return workerExecuteQueue.size(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Kill tasks that have not been executed, like delay task |
||||||
|
* then send Response to Master, update the execution status of task instance |
||||||
|
*/ |
||||||
|
public void killTaskBeforeExecuteByInstanceId(Integer taskInstanceId) { |
||||||
|
workerExecuteQueue.stream() |
||||||
|
.filter(taskExecuteThread -> taskExecuteThread.getTaskExecutionContext().getTaskInstanceId() == taskInstanceId) |
||||||
|
.forEach(workerExecuteQueue::remove); |
||||||
|
sendTaskKillResponse(taskInstanceId); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* kill task before execute , like delay task |
||||||
|
*/ |
||||||
|
private void sendTaskKillResponse(Integer taskInstanceId) { |
||||||
|
TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); |
||||||
|
if (taskExecutionContext == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
TaskExecuteResponseCommand responseCommand = new TaskExecuteResponseCommand(taskExecutionContext.getTaskInstanceId()); |
||||||
|
responseCommand.setStatus(ExecutionStatus.KILL.getCode()); |
||||||
|
ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command(), Event.RESULT); |
||||||
|
taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* submit task |
||||||
|
* |
||||||
|
* @param taskExecuteThread taskExecuteThread |
||||||
|
* @return submit result |
||||||
|
*/ |
||||||
|
public boolean offer(TaskExecuteThread taskExecuteThread) { |
||||||
|
return workerExecuteQueue.offer(taskExecuteThread); |
||||||
|
} |
||||||
|
|
||||||
|
public void start() { |
||||||
|
Thread thread = new Thread(this, this.getClass().getName()); |
||||||
|
thread.start(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
Thread.currentThread().setName("Worker-Execute-Manager-Thread"); |
||||||
|
TaskExecuteThread taskExecuteThread; |
||||||
|
while (Stopper.isRunning()) { |
||||||
|
try { |
||||||
|
taskExecuteThread = workerExecuteQueue.take(); |
||||||
|
workerExecService.submit(taskExecuteThread); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("An unexpected interrupt is happened, " |
||||||
|
+ "the exception will be ignored and this thread will continue to run", e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,195 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.processor; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.common.thread.ThreadUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.FileUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.utils.ChannelUtils; |
||||||
|
import org.apache.dolphinscheduler.remote.utils.JsonSerializer; |
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; |
||||||
|
import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.concurrent.ExecutorService; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.mockito.Mockito; |
||||||
|
import org.powermock.api.mockito.PowerMockito; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* test task execute processor |
||||||
|
*/ |
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({SpringApplicationContext.class, TaskCallbackService.class, WorkerConfig.class, FileUtils.class, |
||||||
|
JsonSerializer.class, JSONUtils.class, ThreadUtils.class, ExecutorService.class, ChannelUtils.class}) |
||||||
|
public class TaskExecuteProcessorTest { |
||||||
|
|
||||||
|
private TaskExecutionContext taskExecutionContext; |
||||||
|
|
||||||
|
private TaskCallbackService taskCallbackService; |
||||||
|
|
||||||
|
private ExecutorService workerExecService; |
||||||
|
|
||||||
|
private WorkerConfig workerConfig; |
||||||
|
|
||||||
|
private Command command; |
||||||
|
|
||||||
|
private Command ackCommand; |
||||||
|
|
||||||
|
private TaskExecuteRequestCommand taskRequestCommand; |
||||||
|
|
||||||
|
private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; |
||||||
|
|
||||||
|
private AlertClientService alertClientService; |
||||||
|
|
||||||
|
private WorkerManagerThread workerManager; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() throws Exception { |
||||||
|
// init task execution context
|
||||||
|
taskExecutionContext = getTaskExecutionContext(); |
||||||
|
workerConfig = new WorkerConfig(); |
||||||
|
workerConfig.setWorkerExecThreads(1); |
||||||
|
workerConfig.setListenPort(1234); |
||||||
|
command = new Command(); |
||||||
|
command.setType(CommandType.TASK_EXECUTE_REQUEST); |
||||||
|
ackCommand = new TaskExecuteAckCommand().convert2Command(); |
||||||
|
taskRequestCommand = new TaskExecuteRequestCommand(); |
||||||
|
alertClientService = PowerMockito.mock(AlertClientService.class); |
||||||
|
workerExecService = PowerMockito.mock(ExecutorService.class); |
||||||
|
PowerMockito.when(workerExecService.submit(Mockito.any(TaskExecuteThread.class))) |
||||||
|
.thenReturn(null); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(ChannelUtils.class); |
||||||
|
PowerMockito.when(ChannelUtils.toAddress(null)).thenReturn(null); |
||||||
|
|
||||||
|
taskExecutionContextCacheManager = PowerMockito.mock(TaskExecutionContextCacheManagerImpl.class); |
||||||
|
taskCallbackService = PowerMockito.mock(TaskCallbackService.class); |
||||||
|
PowerMockito.doNothing().when(taskCallbackService).sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(SpringApplicationContext.class); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskCallbackService.class)) |
||||||
|
.thenReturn(taskCallbackService); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class)) |
||||||
|
.thenReturn(workerConfig); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) |
||||||
|
.thenReturn(null); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) |
||||||
|
.thenReturn(taskExecutionContextCacheManager); |
||||||
|
|
||||||
|
Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, |
||||||
|
taskExecutionContext.getProcessDefineId(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId())); |
||||||
|
|
||||||
|
workerManager = PowerMockito.mock(WorkerManagerThread.class); |
||||||
|
PowerMockito.when(workerManager.offer(new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger, alertClientService))).thenReturn(Boolean.TRUE); |
||||||
|
|
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(WorkerManagerThread.class)) |
||||||
|
.thenReturn(workerManager); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(ThreadUtils.class); |
||||||
|
PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getWorkerExecThreads())) |
||||||
|
.thenReturn(workerExecService); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(JsonSerializer.class); |
||||||
|
PowerMockito.when(JsonSerializer.deserialize(command.getBody(), TaskExecuteRequestCommand.class)) |
||||||
|
.thenReturn(taskRequestCommand); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(JSONUtils.class); |
||||||
|
PowerMockito.when(JSONUtils.parseObject(command.getBody(), TaskExecuteRequestCommand.class)) |
||||||
|
.thenReturn(taskRequestCommand); |
||||||
|
PowerMockito.when(JSONUtils.parseObject(taskRequestCommand.getTaskExecutionContext(), TaskExecutionContext.class)) |
||||||
|
.thenReturn(taskExecutionContext); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(FileUtils.class); |
||||||
|
PowerMockito.when(FileUtils.getProcessExecDir(taskExecutionContext.getProjectId(), |
||||||
|
taskExecutionContext.getProcessDefineId(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId())) |
||||||
|
.thenReturn(taskExecutionContext.getExecutePath()); |
||||||
|
PowerMockito.doNothing().when(FileUtils.class, "createWorkDirIfAbsent", taskExecutionContext.getExecutePath()); |
||||||
|
|
||||||
|
SimpleTaskExecuteThread simpleTaskExecuteThread = new SimpleTaskExecuteThread(null, null, null, alertClientService); |
||||||
|
PowerMockito.whenNew(TaskExecuteThread.class).withAnyArguments() |
||||||
|
.thenReturn(simpleTaskExecuteThread); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testNormalExecution() { |
||||||
|
TaskExecuteProcessor processor = new TaskExecuteProcessor(); |
||||||
|
processor.process(null, command); |
||||||
|
|
||||||
|
Assert.assertEquals(ExecutionStatus.RUNNING_EXECUTION, taskExecutionContext.getCurrentExecutionStatus()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testDelayExecution() { |
||||||
|
taskExecutionContext.setDelayTime(1); |
||||||
|
TaskExecuteProcessor processor = new TaskExecuteProcessor(); |
||||||
|
processor.process(null, command); |
||||||
|
|
||||||
|
Assert.assertEquals(ExecutionStatus.DELAY_EXECUTION, taskExecutionContext.getCurrentExecutionStatus()); |
||||||
|
} |
||||||
|
|
||||||
|
public TaskExecutionContext getTaskExecutionContext() { |
||||||
|
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); |
||||||
|
taskExecutionContext.setProcessId(12345); |
||||||
|
taskExecutionContext.setProcessDefineId(1); |
||||||
|
taskExecutionContext.setProcessInstanceId(1); |
||||||
|
taskExecutionContext.setTaskInstanceId(1); |
||||||
|
taskExecutionContext.setTaskType("sql"); |
||||||
|
taskExecutionContext.setFirstSubmitTime(new Date()); |
||||||
|
taskExecutionContext.setDelayTime(0); |
||||||
|
taskExecutionContext.setLogPath("/tmp/test.log"); |
||||||
|
taskExecutionContext.setHost("localhost"); |
||||||
|
taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); |
||||||
|
return taskExecutionContext; |
||||||
|
} |
||||||
|
|
||||||
|
private static class SimpleTaskExecuteThread extends TaskExecuteThread { |
||||||
|
|
||||||
|
public SimpleTaskExecuteThread(TaskExecutionContext taskExecutionContext, TaskCallbackService taskCallbackService, Logger taskLogger, AlertClientService alertClientService) { |
||||||
|
super(taskExecutionContext, taskCallbackService, taskLogger, alertClientService); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
//
|
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,187 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.common.model.TaskNode; |
||||||
|
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||||
|
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||||
|
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.OSUtils; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.AbstractTask; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.TaskManager; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
|
||||||
|
import java.util.Collections; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.powermock.api.mockito.PowerMockito; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* test worker manager thread. |
||||||
|
*/ |
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({ |
||||||
|
Stopper.class, |
||||||
|
TaskManager.class, |
||||||
|
JSONUtils.class, |
||||||
|
CommonUtils.class, |
||||||
|
SpringApplicationContext.class, |
||||||
|
OSUtils.class}) |
||||||
|
public class WorkerManagerThreadTest { |
||||||
|
|
||||||
|
private TaskCallbackService taskCallbackService; |
||||||
|
|
||||||
|
private WorkerManagerThread workerManager; |
||||||
|
|
||||||
|
private TaskExecutionContext taskExecutionContext; |
||||||
|
|
||||||
|
private AlertClientService alertClientService; |
||||||
|
|
||||||
|
private Logger taskLogger; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
// init task execution context, logger
|
||||||
|
taskExecutionContext = new TaskExecutionContext(); |
||||||
|
taskExecutionContext.setProcessId(12345); |
||||||
|
taskExecutionContext.setProcessDefineId(1); |
||||||
|
taskExecutionContext.setProcessInstanceId(1); |
||||||
|
taskExecutionContext.setTaskInstanceId(1); |
||||||
|
taskExecutionContext.setTenantCode("test"); |
||||||
|
taskExecutionContext.setTaskType(""); |
||||||
|
taskExecutionContext.setFirstSubmitTime(new Date()); |
||||||
|
taskExecutionContext.setDelayTime(0); |
||||||
|
taskExecutionContext.setLogPath("/tmp/test.log"); |
||||||
|
taskExecutionContext.setHost("localhost"); |
||||||
|
taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); |
||||||
|
|
||||||
|
Command ackCommand = new TaskExecuteAckCommand().convert2Command(); |
||||||
|
Command responseCommand = new TaskExecuteResponseCommand(taskExecutionContext.getTaskInstanceId()).convert2Command(); |
||||||
|
|
||||||
|
taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( |
||||||
|
LoggerUtils.TASK_LOGGER_INFO_PREFIX, |
||||||
|
taskExecutionContext.getProcessDefineId(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId() |
||||||
|
)); |
||||||
|
|
||||||
|
TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); |
||||||
|
taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); |
||||||
|
|
||||||
|
alertClientService = PowerMockito.mock(AlertClientService.class); |
||||||
|
WorkerConfig workerConfig = PowerMockito.mock(WorkerConfig.class); |
||||||
|
taskCallbackService = PowerMockito.mock(TaskCallbackService.class); |
||||||
|
PowerMockito.doNothing().when(taskCallbackService).sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); |
||||||
|
PowerMockito.doNothing().when(taskCallbackService).sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand); |
||||||
|
PowerMockito.mockStatic(SpringApplicationContext.class); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) |
||||||
|
.thenReturn(taskExecutionContextCacheManager); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class)) |
||||||
|
.thenReturn(workerConfig); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskCallbackService.class)) |
||||||
|
.thenReturn(taskCallbackService); |
||||||
|
PowerMockito.when(workerConfig.getWorkerExecThreads()).thenReturn(5); |
||||||
|
workerManager = new WorkerManagerThread(); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(TaskManager.class); |
||||||
|
PowerMockito.when(TaskManager.newTask(taskExecutionContext, taskLogger, alertClientService)) |
||||||
|
.thenReturn(new SimpleTask(taskExecutionContext, taskLogger)); |
||||||
|
PowerMockito.mockStatic(JSONUtils.class); |
||||||
|
PowerMockito.when(JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class)) |
||||||
|
.thenReturn(new TaskNode()); |
||||||
|
PowerMockito.mockStatic(CommonUtils.class); |
||||||
|
PowerMockito.when(CommonUtils.getSystemEnvPath()).thenReturn("/user_home/.bash_profile"); |
||||||
|
List<String> osUserList = Collections.singletonList("test"); |
||||||
|
PowerMockito.mockStatic(OSUtils.class); |
||||||
|
PowerMockito.when(OSUtils.getUserList()).thenReturn(osUserList); |
||||||
|
PowerMockito.mockStatic(Stopper.class); |
||||||
|
PowerMockito.when(Stopper.isRunning()).thenReturn(true, false); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSendTaskKillResponse() { |
||||||
|
TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger, alertClientService); |
||||||
|
workerManager.offer(taskExecuteThread); |
||||||
|
Assert.assertEquals(1, workerManager.getQueueSize()); |
||||||
|
workerManager.killTaskBeforeExecuteByInstanceId(1); |
||||||
|
Assert.assertEquals(0, workerManager.getQueueSize()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testRun() { |
||||||
|
TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger, alertClientService); |
||||||
|
workerManager.offer(taskExecuteThread); |
||||||
|
Assert.assertEquals(1, workerManager.getQueueSize()); |
||||||
|
workerManager.run(); |
||||||
|
Assert.assertEquals(0, workerManager.getQueueSize()); |
||||||
|
} |
||||||
|
|
||||||
|
private static class SimpleTask extends AbstractTask { |
||||||
|
|
||||||
|
protected SimpleTask(TaskExecutionContext taskExecutionContext, Logger logger) { |
||||||
|
super(taskExecutionContext, logger); |
||||||
|
// pid
|
||||||
|
this.processId = taskExecutionContext.getProcessId(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public AbstractParameters getParameters() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void init() { |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void handle() { |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void after() { |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public ExecutionStatus getExitStatus() { |
||||||
|
return ExecutionStatus.SUCCESS; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.task; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({SpringApplicationContext.class}) |
||||||
|
public class AbstractCommandExecutorTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(AbstractCommandExecutorTest.class); |
||||||
|
|
||||||
|
private ShellCommandExecutor shellCommandExecutor; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() throws Exception { |
||||||
|
System.setProperty("log4j2.disable.jmx", Boolean.TRUE.toString()); |
||||||
|
shellCommandExecutor = new ShellCommandExecutor(null); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSetTaskResultString() { |
||||||
|
shellCommandExecutor.setTaskResultString("shellReturn"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testGetTaskResultString() { |
||||||
|
logger.info(shellCommandExecutor.getTaskResultString()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,127 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.task; |
||||||
|
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.shell.ShellTaskTest; |
||||||
|
|
||||||
|
import java.nio.file.Files; |
||||||
|
import java.nio.file.Paths; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.powermock.api.mockito.PowerMockito; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* shell task return test. |
||||||
|
*/ |
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({ShellTask.class}) |
||||||
|
public class ShellTaskReturnTest { |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class); |
||||||
|
|
||||||
|
private ShellTask shellTask; |
||||||
|
private ShellCommandExecutor shellCommandExecutor; |
||||||
|
private TaskExecutionContext taskExecutionContext; |
||||||
|
private CommandExecuteResult commandExecuteResult; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() throws Exception { |
||||||
|
System.setProperty("log4j2.disable.jmx", Boolean.TRUE.toString()); |
||||||
|
shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); |
||||||
|
PowerMockito.whenNew(ShellCommandExecutor.class).withAnyArguments().thenReturn(shellCommandExecutor); |
||||||
|
taskExecutionContext = new TaskExecutionContext(); |
||||||
|
taskExecutionContext.setTaskInstanceId(1); |
||||||
|
taskExecutionContext.setTaskName("kris test"); |
||||||
|
taskExecutionContext.setTaskType("SHELL"); |
||||||
|
taskExecutionContext.setHost("127.0.0.1:1234"); |
||||||
|
taskExecutionContext.setExecutePath("/tmp"); |
||||||
|
taskExecutionContext.setLogPath("/log"); |
||||||
|
taskExecutionContext.setTaskJson( |
||||||
|
"{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"]," |
||||||
|
+ "\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false," |
||||||
|
+ "\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"" |
||||||
|
+ "tasks-16849\",\"maxRetryTimes\":0,\"name\":\"shell test 001\"," |
||||||
|
+ "\"params\":\"{\\\"rawScript\\\":\\\"#!/bin/sh\\\\necho $[yyyy-MM-dd HH:mm:ss +3]\\\\necho \\\\\\\" ?? " |
||||||
|
+ "${time1} \\\\\\\"\\\\necho \\\\\\\" ????? ${time2}\\\\\\\"\\\\n\\\"," |
||||||
|
+ "\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":" |
||||||
|
+ "\\\"VARCHAR\\\",\\\"value\\\":\\\"$[yyyy-MM-dd HH:mm:ss]\\\"}," |
||||||
|
+ "{\\\"prop\\\":\\\"time2\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${time_gb}\\\"}" |
||||||
|
+ "],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1," |
||||||
|
+ "\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":" |
||||||
|
+ "{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\"," |
||||||
|
+ "\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); |
||||||
|
taskExecutionContext.setProcessInstanceId(1); |
||||||
|
taskExecutionContext.setGlobalParams("[{\"direct\":\"IN\",\"prop\":\"time_gb\",\"type\":\"VARCHAR\",\"value\":\"2020-12-16 17:18:33\"}]"); |
||||||
|
taskExecutionContext.setExecutorId(1); |
||||||
|
taskExecutionContext.setCmdTypeIfComplement(5); |
||||||
|
taskExecutionContext.setTenantCode("roo"); |
||||||
|
taskExecutionContext.setScheduleTime(new Date()); |
||||||
|
taskExecutionContext.setQueue("default"); |
||||||
|
taskExecutionContext.setTaskParams( |
||||||
|
"{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss +3]\\necho \\\" ?? ${time1} \\\"\\necho \\\" ????? ${time2}\\\"\\n\",\"localParams\":" |
||||||
|
+ |
||||||
|
"[{\"prop\":\"time1\",\"direct\":\"OUT\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR" |
||||||
|
+ "\",\"value\":\"${time_gb}\"}],\"resourceList\":[]}"); |
||||||
|
Map<String, String> definedParams = new HashMap<>(); |
||||||
|
definedParams.put("time_gb", "2020-12-16 00:00:00"); |
||||||
|
taskExecutionContext.setDefinedParams(definedParams); |
||||||
|
PowerMockito.mockStatic(Files.class); |
||||||
|
PowerMockito.when(Files.exists(Paths.get(anyString()))).thenReturn(true); |
||||||
|
commandExecuteResult = new CommandExecuteResult(); |
||||||
|
commandExecuteResult.setAppIds("appId"); |
||||||
|
commandExecuteResult.setExitStatusCode(0); |
||||||
|
commandExecuteResult.setProcessId(1); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testShellReturnString() { |
||||||
|
shellTask = new ShellTask(taskExecutionContext, logger); |
||||||
|
shellTask.init(); |
||||||
|
try { |
||||||
|
PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
shellTask.setResult("shell return string"); |
||||||
|
logger.info("shell return string:{}", shellTask.getResultString()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSetTaskResultString() { |
||||||
|
shellCommandExecutor.setTaskResultString("shellReturn"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testGetTaskResultString() { |
||||||
|
logger.info(shellCommandExecutor.getTaskResultString()); |
||||||
|
} |
||||||
|
} |
||||||
Binary file not shown.
Loading…
Reference in new issue