Wenjun Ruan
2 years ago
committed by
GitHub
39 changed files with 1106 additions and 923 deletions
@ -0,0 +1,60 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskException; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
|
||||||
|
public class DefaultWorkerDelayTaskExecuteRunnable extends WorkerDelayTaskExecuteRunnable { |
||||||
|
|
||||||
|
public DefaultWorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String workflowMaster, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
super(taskExecutionContext, workerConfig, workflowMaster, workerMessageSender, alertClientService, taskPluginManager, storageOperate); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void executeTask() throws TaskException { |
||||||
|
if (task == null) { |
||||||
|
throw new TaskException("The task plugin instance is not initialized"); |
||||||
|
} |
||||||
|
task.handle(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void afterExecute() { |
||||||
|
super.afterExecute(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void afterThrowing(Throwable throwable) throws TaskException { |
||||||
|
super.afterThrowing(throwable); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
|
||||||
|
public class DefaultWorkerDelayTaskExecuteRunnableFactory extends WorkerDelayTaskExecuteRunnableFactory<DefaultWorkerDelayTaskExecuteRunnable> { |
||||||
|
|
||||||
|
protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String workflowMasterAddress, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
super(taskExecutionContext, workerConfig, workflowMasterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public DefaultWorkerDelayTaskExecuteRunnable createWorkerTaskExecuteRunnable() { |
||||||
|
return new DefaultWorkerDelayTaskExecuteRunnable( |
||||||
|
taskExecutionContext, |
||||||
|
workerConfig, |
||||||
|
workflowMasterAddress, |
||||||
|
workerMessageSender, |
||||||
|
alertClientService, |
||||||
|
taskPluginManager, |
||||||
|
storageOperate); |
||||||
|
} |
||||||
|
} |
@ -1,364 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.server.worker.runner; |
|
||||||
|
|
||||||
import com.google.common.base.Strings; |
|
||||||
import lombok.NonNull; |
|
||||||
import org.apache.commons.collections.MapUtils; |
|
||||||
import org.apache.commons.lang3.tuple.Pair; |
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.enums.WarningType; |
|
||||||
import org.apache.dolphinscheduler.common.exception.StorageOperateNoConfiguredException; |
|
||||||
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
|
||||||
import org.apache.dolphinscheduler.common.utils.*; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; |
|
||||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
|
||||||
import org.apache.dolphinscheduler.server.utils.ProcessUtils; |
|
||||||
import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; |
|
||||||
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
|
||||||
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
|
||||||
import org.apache.dolphinscheduler.service.exceptions.ServiceException; |
|
||||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import java.io.File; |
|
||||||
import java.io.IOException; |
|
||||||
import java.nio.file.Files; |
|
||||||
import java.nio.file.NoSuchFileException; |
|
||||||
import java.nio.file.Paths; |
|
||||||
import java.util.*; |
|
||||||
import java.util.concurrent.Delayed; |
|
||||||
import java.util.concurrent.TimeUnit; |
|
||||||
|
|
||||||
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; |
|
||||||
|
|
||||||
/** |
|
||||||
* task scheduler thread |
|
||||||
*/ |
|
||||||
public class TaskExecuteThread implements Runnable, Delayed { |
|
||||||
|
|
||||||
/** |
|
||||||
* logger |
|
||||||
*/ |
|
||||||
private final Logger logger = LoggerFactory.getLogger(TaskExecuteThread.class); |
|
||||||
|
|
||||||
/** |
|
||||||
* task instance |
|
||||||
*/ |
|
||||||
private final TaskExecutionContext taskExecutionContext; |
|
||||||
|
|
||||||
private final String masterAddress; |
|
||||||
|
|
||||||
private final StorageOperate storageOperate; |
|
||||||
|
|
||||||
/** |
|
||||||
* abstract task |
|
||||||
*/ |
|
||||||
private AbstractTask task; |
|
||||||
|
|
||||||
/** |
|
||||||
* task callback service |
|
||||||
*/ |
|
||||||
private final WorkerMessageSender workerMessageSender; |
|
||||||
|
|
||||||
/** |
|
||||||
* alert client server |
|
||||||
*/ |
|
||||||
private final AlertClientService alertClientService; |
|
||||||
|
|
||||||
private TaskPluginManager taskPluginManager; |
|
||||||
|
|
||||||
/** |
|
||||||
* constructor |
|
||||||
* |
|
||||||
* @param taskExecutionContext taskExecutionContext |
|
||||||
* @param workerMessageSender used for worker send message to master |
|
||||||
*/ |
|
||||||
public TaskExecuteThread(@NonNull TaskExecutionContext taskExecutionContext, |
|
||||||
@NonNull String masterAddress, |
|
||||||
@NonNull WorkerMessageSender workerMessageSender, |
|
||||||
@NonNull AlertClientService alertClientService, |
|
||||||
StorageOperate storageOperate) { |
|
||||||
this.taskExecutionContext = taskExecutionContext; |
|
||||||
this.masterAddress = masterAddress; |
|
||||||
this.workerMessageSender = workerMessageSender; |
|
||||||
this.alertClientService = alertClientService; |
|
||||||
this.storageOperate = storageOperate; |
|
||||||
} |
|
||||||
|
|
||||||
public TaskExecuteThread(@NonNull TaskExecutionContext taskExecutionContext, |
|
||||||
@NonNull String masterAddress, |
|
||||||
@NonNull WorkerMessageSender workerMessageSender, |
|
||||||
@NonNull AlertClientService alertClientService, |
|
||||||
@NonNull TaskPluginManager taskPluginManager, |
|
||||||
StorageOperate storageOperate) { |
|
||||||
this.taskExecutionContext = taskExecutionContext; |
|
||||||
this.masterAddress = masterAddress; |
|
||||||
this.workerMessageSender = workerMessageSender; |
|
||||||
this.alertClientService = alertClientService; |
|
||||||
this.taskPluginManager = taskPluginManager; |
|
||||||
this.storageOperate = storageOperate; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public void run() { |
|
||||||
try { |
|
||||||
LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), |
|
||||||
taskExecutionContext.getTaskInstanceId()); |
|
||||||
if (Constants.DRY_RUN_FLAG_YES == taskExecutionContext.getDryRun()) { |
|
||||||
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); |
|
||||||
taskExecutionContext.setStartTime(new Date()); |
|
||||||
taskExecutionContext.setEndTime(new Date()); |
|
||||||
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); |
|
||||||
workerMessageSender.sendMessageWithRetry(taskExecutionContext, |
|
||||||
masterAddress, |
|
||||||
CommandType.TASK_EXECUTE_RESULT); |
|
||||||
logger.info("Task dry run success"); |
|
||||||
return; |
|
||||||
} |
|
||||||
} finally { |
|
||||||
LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); |
|
||||||
} |
|
||||||
try { |
|
||||||
LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), |
|
||||||
taskExecutionContext.getTaskInstanceId()); |
|
||||||
logger.info("script path : {}", taskExecutionContext.getExecutePath()); |
|
||||||
if (taskExecutionContext.getStartTime() == null) { |
|
||||||
taskExecutionContext.setStartTime(new Date()); |
|
||||||
} |
|
||||||
logger.info("the task begins to execute. task instance id: {}", taskExecutionContext.getTaskInstanceId()); |
|
||||||
|
|
||||||
// callback task execute running
|
|
||||||
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.RUNNING_EXECUTION); |
|
||||||
workerMessageSender.sendMessageWithRetry(taskExecutionContext, |
|
||||||
masterAddress, |
|
||||||
CommandType.TASK_EXECUTE_RUNNING); |
|
||||||
|
|
||||||
// copy hdfs/minio file to local
|
|
||||||
List<Pair<String, String>> fileDownloads = downloadCheck(taskExecutionContext.getExecutePath(), |
|
||||||
taskExecutionContext.getResources()); |
|
||||||
if (!fileDownloads.isEmpty()) { |
|
||||||
downloadResource(taskExecutionContext.getExecutePath(), logger, fileDownloads); |
|
||||||
} |
|
||||||
|
|
||||||
taskExecutionContext.setEnvFile(CommonUtils.getSystemEnvPath()); |
|
||||||
|
|
||||||
taskExecutionContext.setTaskAppId(String.format("%s_%s", |
|
||||||
taskExecutionContext.getProcessInstanceId(), |
|
||||||
taskExecutionContext.getTaskInstanceId())); |
|
||||||
|
|
||||||
TaskChannel taskChannel = taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType()); |
|
||||||
if (null == taskChannel) { |
|
||||||
throw new ServiceException(String.format("%s Task Plugin Not Found,Please Check Config File.", |
|
||||||
taskExecutionContext.getTaskType())); |
|
||||||
} |
|
||||||
String taskLogName = LoggerUtils.buildTaskId(taskExecutionContext.getFirstSubmitTime(), |
|
||||||
taskExecutionContext.getProcessDefineCode(), |
|
||||||
taskExecutionContext.getProcessDefineVersion(), |
|
||||||
taskExecutionContext.getProcessInstanceId(), |
|
||||||
taskExecutionContext.getTaskInstanceId()); |
|
||||||
taskExecutionContext.setTaskLogName(taskLogName); |
|
||||||
|
|
||||||
// set the name of the current thread
|
|
||||||
Thread.currentThread().setName(taskLogName); |
|
||||||
|
|
||||||
task = taskChannel.createTask(taskExecutionContext); |
|
||||||
|
|
||||||
// task init
|
|
||||||
this.task.init(); |
|
||||||
|
|
||||||
// init varPool
|
|
||||||
this.task.getParameters().setVarPool(taskExecutionContext.getVarPool()); |
|
||||||
|
|
||||||
// task handle
|
|
||||||
this.task.handle(); |
|
||||||
|
|
||||||
// task result process
|
|
||||||
if (this.task.getNeedAlert()) { |
|
||||||
sendAlert(this.task.getTaskAlertInfo(), this.task.getExitStatus()); |
|
||||||
} |
|
||||||
|
|
||||||
taskExecutionContext.setCurrentExecutionStatus(this.task.getExitStatus()); |
|
||||||
taskExecutionContext.setEndTime(DateUtils.getCurrentDate()); |
|
||||||
taskExecutionContext.setProcessId(this.task.getProcessId()); |
|
||||||
taskExecutionContext.setAppIds(this.task.getAppIds()); |
|
||||||
taskExecutionContext.setVarPool(JSONUtils.toJsonString(this.task.getParameters().getVarPool())); |
|
||||||
logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), |
|
||||||
this.task.getExitStatus()); |
|
||||||
} catch (Throwable e) { |
|
||||||
logger.error("task scheduler failure", e); |
|
||||||
kill(); |
|
||||||
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.FAILURE); |
|
||||||
taskExecutionContext.setEndTime(DateUtils.getCurrentDate()); |
|
||||||
taskExecutionContext.setProcessId(this.task.getProcessId()); |
|
||||||
taskExecutionContext.setAppIds(this.task.getAppIds()); |
|
||||||
} finally { |
|
||||||
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); |
|
||||||
workerMessageSender.sendMessageWithRetry(taskExecutionContext, |
|
||||||
masterAddress, |
|
||||||
CommandType.TASK_EXECUTE_RESULT); |
|
||||||
clearTaskExecPath(); |
|
||||||
LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private void sendAlert(TaskAlertInfo taskAlertInfo, TaskExecutionStatus status) { |
|
||||||
int strategy = |
|
||||||
status == TaskExecutionStatus.SUCCESS ? WarningType.SUCCESS.getCode() : WarningType.FAILURE.getCode(); |
|
||||||
alertClientService.sendAlert(taskAlertInfo.getAlertGroupId(), taskAlertInfo.getTitle(), |
|
||||||
taskAlertInfo.getContent(), strategy); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* when task finish, clear execute path. |
|
||||||
*/ |
|
||||||
private void clearTaskExecPath() { |
|
||||||
logger.info("develop mode is: {}", CommonUtils.isDevelopMode()); |
|
||||||
|
|
||||||
if (!CommonUtils.isDevelopMode()) { |
|
||||||
// get exec dir
|
|
||||||
String execLocalPath = taskExecutionContext.getExecutePath(); |
|
||||||
|
|
||||||
if (Strings.isNullOrEmpty(execLocalPath)) { |
|
||||||
logger.warn("task: {} exec local path is empty.", taskExecutionContext.getTaskName()); |
|
||||||
return; |
|
||||||
} |
|
||||||
|
|
||||||
if (SINGLE_SLASH.equals(execLocalPath)) { |
|
||||||
logger.warn("task: {} exec local path is '/', direct deletion is not allowed", |
|
||||||
taskExecutionContext.getTaskName()); |
|
||||||
return; |
|
||||||
} |
|
||||||
|
|
||||||
try { |
|
||||||
org.apache.commons.io.FileUtils.deleteDirectory(new File(execLocalPath)); |
|
||||||
logger.info("exec local path: {} cleared.", execLocalPath); |
|
||||||
} catch (IOException e) { |
|
||||||
if (e instanceof NoSuchFileException) { |
|
||||||
// this is expected
|
|
||||||
} else { |
|
||||||
logger.error("Delete exec dir failed.", e); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* kill task |
|
||||||
*/ |
|
||||||
public void kill() { |
|
||||||
if (task != null) { |
|
||||||
try { |
|
||||||
task.cancelApplication(true); |
|
||||||
ProcessUtils.killYarnJob(taskExecutionContext); |
|
||||||
} catch (Exception e) { |
|
||||||
logger.error("Kill task failed", e); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* download resource file |
|
||||||
* |
|
||||||
* @param execLocalPath execLocalPath |
|
||||||
* @param fileDownloads projectRes |
|
||||||
* @param logger logger |
|
||||||
*/ |
|
||||||
public void downloadResource(String execLocalPath, Logger logger, List<Pair<String, String>> fileDownloads) { |
|
||||||
for (Pair<String, String> fileDownload : fileDownloads) { |
|
||||||
try { |
|
||||||
// query the tenant code of the resource according to the name of the resource
|
|
||||||
String fullName = fileDownload.getLeft(); |
|
||||||
String tenantCode = fileDownload.getRight(); |
|
||||||
String resPath = storageOperate.getResourceFileName(tenantCode, fullName); |
|
||||||
logger.info("get resource file from path:{}", resPath); |
|
||||||
long resourceDownloadStartTime = System.currentTimeMillis(); |
|
||||||
storageOperate.download(tenantCode, resPath, execLocalPath + File.separator + fullName, false, true); |
|
||||||
WorkerServerMetrics |
|
||||||
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime); |
|
||||||
WorkerServerMetrics.recordWorkerResourceDownloadSize( |
|
||||||
Files.size(Paths.get(execLocalPath, fullName))); |
|
||||||
WorkerServerMetrics.incWorkerResourceDownloadSuccessCount(); |
|
||||||
} catch (Exception e) { |
|
||||||
WorkerServerMetrics.incWorkerResourceDownloadFailureCount(); |
|
||||||
logger.error(e.getMessage(), e); |
|
||||||
throw new ServiceException(e.getMessage()); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* download resource check |
|
||||||
* |
|
||||||
* @param execLocalPath |
|
||||||
* @param projectRes |
|
||||||
* @return |
|
||||||
*/ |
|
||||||
public List<Pair<String, String>> downloadCheck(String execLocalPath, Map<String, String> projectRes) { |
|
||||||
if (MapUtils.isEmpty(projectRes)) { |
|
||||||
return Collections.emptyList(); |
|
||||||
} |
|
||||||
List<Pair<String, String>> downloadFile = new ArrayList<>(); |
|
||||||
projectRes.forEach((key, value) -> { |
|
||||||
File resFile = new File(execLocalPath, key); |
|
||||||
boolean notExist = !resFile.exists(); |
|
||||||
if (notExist) { |
|
||||||
downloadFile.add(Pair.of(key, value)); |
|
||||||
} else { |
|
||||||
logger.info("file : {} exists ", resFile.getName()); |
|
||||||
} |
|
||||||
}); |
|
||||||
if (!downloadFile.isEmpty() && !PropertyUtils.getResUploadStartupState()) { |
|
||||||
throw new StorageOperateNoConfiguredException("Storage service config does not exist!"); |
|
||||||
} |
|
||||||
return downloadFile; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get current TaskExecutionContext |
|
||||||
* |
|
||||||
* @return TaskExecutionContext |
|
||||||
*/ |
|
||||||
public TaskExecutionContext getTaskExecutionContext() { |
|
||||||
return this.taskExecutionContext; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public long getDelay(TimeUnit unit) { |
|
||||||
return unit.convert(DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), |
|
||||||
taskExecutionContext.getDelayTime() * 60L), TimeUnit.SECONDS); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public int compareTo(Delayed o) { |
|
||||||
if (o == null) { |
|
||||||
return 1; |
|
||||||
} |
|
||||||
return Long.compare(this.getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); |
|
||||||
} |
|
||||||
|
|
||||||
public AbstractTask getTask() { |
|
||||||
return task; |
|
||||||
} |
|
||||||
} |
|
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
import java.util.concurrent.Delayed; |
||||||
|
import java.util.concurrent.TimeUnit; |
||||||
|
|
||||||
|
public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRunnable implements Delayed { |
||||||
|
|
||||||
|
protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String masterAddress, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
super(taskExecutionContext, workerConfig, masterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getDelay(TimeUnit unit) { |
||||||
|
TaskExecutionContext taskExecutionContext = getTaskExecutionContext(); |
||||||
|
return unit.convert( |
||||||
|
DateUtils.getRemainTime( |
||||||
|
taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L), TimeUnit.SECONDS); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int compareTo(Delayed o) { |
||||||
|
if (o == null) { |
||||||
|
return 1; |
||||||
|
} |
||||||
|
return Long.compare(this.getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,59 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
|
||||||
|
public abstract class WorkerDelayTaskExecuteRunnableFactory<T extends WorkerDelayTaskExecuteRunnable> implements WorkerTaskExecuteRunnableFactory<T> { |
||||||
|
|
||||||
|
protected final @NonNull TaskExecutionContext taskExecutionContext; |
||||||
|
protected final @NonNull WorkerConfig workerConfig; |
||||||
|
protected final @NonNull String workflowMasterAddress; |
||||||
|
protected final @NonNull WorkerMessageSender workerMessageSender; |
||||||
|
protected final @NonNull AlertClientService alertClientService; |
||||||
|
protected final @NonNull TaskPluginManager taskPluginManager; |
||||||
|
protected final @Nullable StorageOperate storageOperate; |
||||||
|
|
||||||
|
protected WorkerDelayTaskExecuteRunnableFactory( |
||||||
|
@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String workflowMasterAddress, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
this.taskExecutionContext = taskExecutionContext; |
||||||
|
this.workerConfig = workerConfig; |
||||||
|
this.workflowMasterAddress = workflowMasterAddress; |
||||||
|
this.workerMessageSender = workerMessageSender; |
||||||
|
this.alertClientService = alertClientService; |
||||||
|
this.taskPluginManager = taskPluginManager; |
||||||
|
this.storageOperate = storageOperate; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public abstract T createWorkerTaskExecuteRunnable(); |
||||||
|
} |
@ -0,0 +1,275 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import com.google.common.base.Strings; |
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.WarningType; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskException; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginException; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.server.utils.ProcessUtils; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionCheckerUtils; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
import java.io.File; |
||||||
|
import java.io.IOException; |
||||||
|
import java.nio.file.NoSuchFileException; |
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; |
||||||
|
|
||||||
|
public abstract class WorkerTaskExecuteRunnable implements Runnable { |
||||||
|
|
||||||
|
protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, WorkerTaskExecuteRunnable.class)); |
||||||
|
|
||||||
|
protected final TaskExecutionContext taskExecutionContext; |
||||||
|
protected final WorkerConfig workerConfig; |
||||||
|
protected final String masterAddress; |
||||||
|
protected final WorkerMessageSender workerMessageSender; |
||||||
|
protected final AlertClientService alertClientService; |
||||||
|
protected final TaskPluginManager taskPluginManager; |
||||||
|
protected final @Nullable StorageOperate storageOperate; |
||||||
|
|
||||||
|
protected @Nullable AbstractTask task; |
||||||
|
|
||||||
|
protected WorkerTaskExecuteRunnable( |
||||||
|
@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String masterAddress, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
this.taskExecutionContext = taskExecutionContext; |
||||||
|
this.workerConfig = workerConfig; |
||||||
|
this.masterAddress = masterAddress; |
||||||
|
this.workerMessageSender = workerMessageSender; |
||||||
|
this.alertClientService = alertClientService; |
||||||
|
this.taskPluginManager = taskPluginManager; |
||||||
|
this.storageOperate = storageOperate; |
||||||
|
String taskLogName = LoggerUtils.buildTaskId(taskExecutionContext.getFirstSubmitTime(), |
||||||
|
taskExecutionContext.getProcessDefineCode(), |
||||||
|
taskExecutionContext.getProcessDefineVersion(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId()); |
||||||
|
taskExecutionContext.setTaskLogName(taskLogName); |
||||||
|
logger.info("Set task logger name: {}", taskLogName); |
||||||
|
} |
||||||
|
|
||||||
|
protected abstract void executeTask(); |
||||||
|
|
||||||
|
protected void afterExecute() throws TaskException { |
||||||
|
if (task == null) { |
||||||
|
throw new TaskException("The current task instance is null"); |
||||||
|
} |
||||||
|
sendAlertIfNeeded(); |
||||||
|
|
||||||
|
sendTaskResult(); |
||||||
|
|
||||||
|
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); |
||||||
|
logger.info("Remove the current task execute context from worker cache"); |
||||||
|
clearTaskExecPathIfNeeded(); |
||||||
|
} |
||||||
|
|
||||||
|
protected void afterThrowing(Throwable throwable) throws TaskException { |
||||||
|
cancelTask(); |
||||||
|
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); |
||||||
|
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.FAILURE); |
||||||
|
taskExecutionContext.setEndTime(new Date()); |
||||||
|
workerMessageSender.sendMessageWithRetry(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RESULT); |
||||||
|
logger.info("Get a exception when execute the task, will send the task execute result to master, the current task execute result is {}", TaskExecutionStatus.FAILURE); |
||||||
|
} |
||||||
|
|
||||||
|
public void cancelTask() { |
||||||
|
// cancel the task
|
||||||
|
if (task != null) { |
||||||
|
try { |
||||||
|
task.cancelApplication(true); |
||||||
|
ProcessUtils.killYarnJob(taskExecutionContext); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("Task execute failed and cancel the application failed, this will not affect the taskInstance status, but you need to check manual", e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
try { |
||||||
|
// set the thread name to make sure the log be written to the task log file
|
||||||
|
Thread.currentThread().setName(taskExecutionContext.getTaskLogName()); |
||||||
|
|
||||||
|
LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); |
||||||
|
logger.info("Begin to pulling task"); |
||||||
|
|
||||||
|
initializeTask(); |
||||||
|
|
||||||
|
if (Constants.DRY_RUN_FLAG_YES == taskExecutionContext.getDryRun()) { |
||||||
|
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); |
||||||
|
taskExecutionContext.setEndTime(new Date()); |
||||||
|
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); |
||||||
|
workerMessageSender.sendMessageWithRetry(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RESULT); |
||||||
|
logger.info("The current execute mode is dry run, will stop the subsequent process and set the taskInstance status to success"); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
beforeExecute(); |
||||||
|
|
||||||
|
executeTask(); |
||||||
|
|
||||||
|
afterExecute(); |
||||||
|
|
||||||
|
} catch (Throwable ex) { |
||||||
|
logger.error("Task execute failed, due to meet an exception", ex); |
||||||
|
afterThrowing(ex); |
||||||
|
} finally { |
||||||
|
LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
protected void initializeTask() { |
||||||
|
logger.info("Begin to initialize task"); |
||||||
|
|
||||||
|
Date taskStartTime = new Date(); |
||||||
|
taskExecutionContext.setStartTime(taskStartTime); |
||||||
|
logger.info("Set task startTime: {}", taskStartTime); |
||||||
|
|
||||||
|
String systemEnvPath = CommonUtils.getSystemEnvPath(); |
||||||
|
taskExecutionContext.setEnvFile(systemEnvPath); |
||||||
|
logger.info("Set task envFile: {}", systemEnvPath); |
||||||
|
|
||||||
|
String taskAppId = String.format("%s_%s", taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); |
||||||
|
taskExecutionContext.setTaskAppId(taskAppId); |
||||||
|
logger.info("Set task appId: {}", taskAppId); |
||||||
|
|
||||||
|
logger.info("End initialize task"); |
||||||
|
} |
||||||
|
|
||||||
|
protected void beforeExecute() { |
||||||
|
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.RUNNING_EXECUTION); |
||||||
|
workerMessageSender.sendMessageWithRetry(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RUNNING); |
||||||
|
logger.info("Set task status to {}", TaskExecutionStatus.RUNNING_EXECUTION); |
||||||
|
|
||||||
|
TaskExecutionCheckerUtils.checkTenantExist(workerConfig, taskExecutionContext); |
||||||
|
logger.info("TenantCode:{} check success", taskExecutionContext.getTenantCode()); |
||||||
|
|
||||||
|
TaskExecutionCheckerUtils.createProcessLocalPathIfAbsent(taskExecutionContext); |
||||||
|
logger.info("ProcessExecDir:{} check success", taskExecutionContext.getExecutePath()); |
||||||
|
|
||||||
|
TaskExecutionCheckerUtils.downloadResourcesIfNeeded(storageOperate, taskExecutionContext, logger); |
||||||
|
logger.info("Resources:{} check success", taskExecutionContext.getResources()); |
||||||
|
|
||||||
|
TaskChannel taskChannel = taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType()); |
||||||
|
if (null == taskChannel) { |
||||||
|
throw new TaskPluginException(String.format("%s task plugin not found, please check config file.", taskExecutionContext.getTaskType())); |
||||||
|
} |
||||||
|
task = taskChannel.createTask(taskExecutionContext); |
||||||
|
if (task == null) { |
||||||
|
throw new TaskPluginException(String.format("%s task is null, please check the task plugin is correct", taskExecutionContext.getTaskType())); |
||||||
|
} |
||||||
|
logger.info("Task plugin: {} create success", taskExecutionContext.getTaskType()); |
||||||
|
|
||||||
|
task.init(); |
||||||
|
logger.info("Success initialized task plugin instance success"); |
||||||
|
|
||||||
|
task.getParameters().setVarPool(taskExecutionContext.getVarPool()); |
||||||
|
logger.info("Success set taskVarPool: {}", taskExecutionContext.getVarPool()); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
protected void sendAlertIfNeeded() { |
||||||
|
if (!task.getNeedAlert()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
logger.info("The current task need to send alert, begin to send alert"); |
||||||
|
TaskExecutionStatus status = task.getExitStatus(); |
||||||
|
TaskAlertInfo taskAlertInfo = task.getTaskAlertInfo(); |
||||||
|
int strategy = status == TaskExecutionStatus.SUCCESS ? WarningType.SUCCESS.getCode() : WarningType.FAILURE.getCode(); |
||||||
|
alertClientService.sendAlert(taskAlertInfo.getAlertGroupId(), taskAlertInfo.getTitle(), taskAlertInfo.getContent(), strategy); |
||||||
|
logger.info("Success send alert"); |
||||||
|
} |
||||||
|
|
||||||
|
protected void sendTaskResult() { |
||||||
|
taskExecutionContext.setCurrentExecutionStatus(task.getExitStatus()); |
||||||
|
taskExecutionContext.setEndTime(new Date()); |
||||||
|
taskExecutionContext.setProcessId(task.getProcessId()); |
||||||
|
taskExecutionContext.setAppIds(task.getAppIds()); |
||||||
|
taskExecutionContext.setVarPool(JSONUtils.toJsonString(task.getParameters().getVarPool())); |
||||||
|
workerMessageSender.sendMessageWithRetry(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RESULT); |
||||||
|
|
||||||
|
logger.info("Send task execute result to master, the current task status: {}", taskExecutionContext.getCurrentExecutionStatus()); |
||||||
|
} |
||||||
|
|
||||||
|
protected void clearTaskExecPathIfNeeded() { |
||||||
|
|
||||||
|
String execLocalPath = taskExecutionContext.getExecutePath(); |
||||||
|
if (!CommonUtils.isDevelopMode()) { |
||||||
|
logger.info("The current execute mode isn't develop mode, will clear the task execute file: {}", execLocalPath); |
||||||
|
// get exec dir
|
||||||
|
if (Strings.isNullOrEmpty(execLocalPath)) { |
||||||
|
logger.warn("The task execute file is {} no need to clear", taskExecutionContext.getTaskName()); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
if (SINGLE_SLASH.equals(execLocalPath)) { |
||||||
|
logger.warn("The task execute file is '/', direct deletion is not allowed"); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
org.apache.commons.io.FileUtils.deleteDirectory(new File(execLocalPath)); |
||||||
|
logger.info("Success clear the task execute file: {}", execLocalPath); |
||||||
|
} catch (IOException e) { |
||||||
|
if (e instanceof NoSuchFileException) { |
||||||
|
// this is expected
|
||||||
|
} else { |
||||||
|
logger.error("Delete task execute file: {} failed, this will not affect the task status, but you need to clear this manually", execLocalPath, e); |
||||||
|
} |
||||||
|
} |
||||||
|
} else { |
||||||
|
logger.info("The current execute mode is develop mode, will not clear the task execute file: {}", execLocalPath); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public @NonNull TaskExecutionContext getTaskExecutionContext() { |
||||||
|
return taskExecutionContext; |
||||||
|
} |
||||||
|
|
||||||
|
public @Nullable AbstractTask getTask() { |
||||||
|
return task; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,23 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
public interface WorkerTaskExecuteRunnableFactory<T> { |
||||||
|
|
||||||
|
T createWorkerTaskExecuteRunnable(); |
||||||
|
} |
@ -0,0 +1,50 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
|
||||||
|
import javax.annotation.Nullable; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class WorkerTaskExecuteRunnableFactoryBuilder { |
||||||
|
|
||||||
|
public static WorkerDelayTaskExecuteRunnableFactory<?> createWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, |
||||||
|
@NonNull WorkerConfig workerConfig, |
||||||
|
@NonNull String workflowMasterAddress, |
||||||
|
@NonNull WorkerMessageSender workerMessageSender, |
||||||
|
@NonNull AlertClientService alertClientService, |
||||||
|
@NonNull TaskPluginManager taskPluginManager, |
||||||
|
@Nullable StorageOperate storageOperate) { |
||||||
|
return new DefaultWorkerDelayTaskExecuteRunnableFactory(taskExecutionContext, |
||||||
|
workerConfig, |
||||||
|
workflowMasterAddress, |
||||||
|
workerMessageSender, |
||||||
|
alertClientService, |
||||||
|
taskPluginManager, |
||||||
|
storageOperate); |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,129 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.utils; |
||||||
|
|
||||||
|
import org.apache.commons.collections.CollectionUtils; |
||||||
|
import org.apache.commons.collections.MapUtils; |
||||||
|
import org.apache.commons.lang.SystemUtils; |
||||||
|
import org.apache.commons.lang3.tuple.Pair; |
||||||
|
import org.apache.dolphinscheduler.common.exception.StorageOperateNoConfiguredException; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.FileUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.OSUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskException; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; |
||||||
|
import org.slf4j.Logger; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
import java.nio.file.Files; |
||||||
|
import java.nio.file.Paths; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
public class TaskExecutionCheckerUtils { |
||||||
|
|
||||||
|
public static void checkTenantExist(WorkerConfig workerConfig, TaskExecutionContext taskExecutionContext) { |
||||||
|
try { |
||||||
|
boolean osUserExistFlag; |
||||||
|
// if Using distributed is true and Currently supported systems are linux,Should not let it
|
||||||
|
// automatically
|
||||||
|
// create tenants,so TenantAutoCreate has no effect
|
||||||
|
if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX) { |
||||||
|
// use the id command to judge in linux
|
||||||
|
osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode()); |
||||||
|
} else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) { |
||||||
|
// if not exists this user, then create
|
||||||
|
OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode()); |
||||||
|
osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); |
||||||
|
} else { |
||||||
|
osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); |
||||||
|
} |
||||||
|
if (!osUserExistFlag) { |
||||||
|
throw new TaskException(String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode())); |
||||||
|
} |
||||||
|
} catch (TaskException ex) { |
||||||
|
throw ex; |
||||||
|
} catch (Exception ex) { |
||||||
|
throw new TaskException(String.format("TenantCode: %s doesn't exist", taskExecutionContext.getTenantCode())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public static void createProcessLocalPathIfAbsent(TaskExecutionContext taskExecutionContext) throws TaskException { |
||||||
|
try { |
||||||
|
// local execute path
|
||||||
|
String execLocalPath = FileUtils.getProcessExecDir( |
||||||
|
taskExecutionContext.getProjectCode(), |
||||||
|
taskExecutionContext.getProcessDefineCode(), |
||||||
|
taskExecutionContext.getProcessDefineVersion(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId()); |
||||||
|
taskExecutionContext.setExecutePath(execLocalPath); |
||||||
|
FileUtils.createWorkDirIfAbsent(execLocalPath); |
||||||
|
} catch (Throwable ex) { |
||||||
|
throw new TaskException("Cannot create process execute dir", ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public static void downloadResourcesIfNeeded(StorageOperate storageOperate, TaskExecutionContext taskExecutionContext, Logger logger) { |
||||||
|
String execLocalPath = taskExecutionContext.getExecutePath(); |
||||||
|
Map<String, String> projectRes = taskExecutionContext.getResources(); |
||||||
|
if (MapUtils.isEmpty(projectRes)) { |
||||||
|
return; |
||||||
|
} |
||||||
|
List<Pair<String, String>> downloadFiles = new ArrayList<>(); |
||||||
|
projectRes.forEach((key, value) -> { |
||||||
|
File resFile = new File(execLocalPath, key); |
||||||
|
boolean notExist = !resFile.exists(); |
||||||
|
if (notExist) { |
||||||
|
downloadFiles.add(Pair.of(key, value)); |
||||||
|
} else { |
||||||
|
logger.info("file : {} exists ", resFile.getName()); |
||||||
|
} |
||||||
|
}); |
||||||
|
if (!downloadFiles.isEmpty() && !PropertyUtils.getResUploadStartupState()) { |
||||||
|
throw new StorageOperateNoConfiguredException("Storage service config does not exist!"); |
||||||
|
} |
||||||
|
|
||||||
|
if (CollectionUtils.isNotEmpty(downloadFiles)) { |
||||||
|
for (Pair<String, String> fileDownload : downloadFiles) { |
||||||
|
try { |
||||||
|
// query the tenant code of the resource according to the name of the resource
|
||||||
|
String fullName = fileDownload.getLeft(); |
||||||
|
String tenantCode = fileDownload.getRight(); |
||||||
|
String resPath = storageOperate.getResourceFileName(tenantCode, fullName); |
||||||
|
logger.info("get resource file from path:{}", resPath); |
||||||
|
long resourceDownloadStartTime = System.currentTimeMillis(); |
||||||
|
storageOperate.download(tenantCode, resPath, execLocalPath + File.separator + fullName, false, true); |
||||||
|
WorkerServerMetrics |
||||||
|
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime); |
||||||
|
WorkerServerMetrics.recordWorkerResourceDownloadSize( |
||||||
|
Files.size(Paths.get(execLocalPath, fullName))); |
||||||
|
WorkerServerMetrics.incWorkerResourceDownloadSuccessCount(); |
||||||
|
} catch (Exception e) { |
||||||
|
WorkerServerMetrics.incWorkerResourceDownloadFailureCount(); |
||||||
|
throw new TaskException(String.format("Download resource file: %s error", fileDownload), e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,73 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.runner; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||||
|
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||||
|
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
||||||
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.jupiter.api.Assertions; |
||||||
|
import org.mockito.Mockito; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
public class DefaultWorkerDelayTaskExecuteRunnableTest { |
||||||
|
|
||||||
|
private TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); |
||||||
|
|
||||||
|
private WorkerConfig workerConfig = Mockito.mock(WorkerConfig.class); |
||||||
|
|
||||||
|
private String masterAddress = "localhost:5678"; |
||||||
|
|
||||||
|
private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); |
||||||
|
|
||||||
|
private AlertClientService alertClientService = Mockito.mock(AlertClientService.class); |
||||||
|
|
||||||
|
private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); |
||||||
|
|
||||||
|
private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testDryRun() { |
||||||
|
TaskExecutionContext taskExecutionContext = TaskExecutionContext.builder() |
||||||
|
.dryRun(Constants.DRY_RUN_FLAG_YES) |
||||||
|
.taskInstanceId(0) |
||||||
|
.processDefineId(0) |
||||||
|
.firstSubmitTime(new Date()) |
||||||
|
.taskLogName("TestLogName") |
||||||
|
.build(); |
||||||
|
WorkerTaskExecuteRunnable workerTaskExecuteRunnable = new DefaultWorkerDelayTaskExecuteRunnable( |
||||||
|
taskExecutionContext, |
||||||
|
workerConfig, |
||||||
|
masterAddress, |
||||||
|
workerMessageSender, |
||||||
|
alertClientService, |
||||||
|
taskPluginManager, |
||||||
|
storageOperate |
||||||
|
); |
||||||
|
|
||||||
|
Assertions.assertAll(workerTaskExecuteRunnable::run); |
||||||
|
Assertions.assertEquals(TaskExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -1,87 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.server.worker.runner; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.storage.StorageOperate; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
||||||
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClientTest; |
|
||||||
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
|
||||||
import org.apache.dolphinscheduler.service.alert.AlertClientService; |
|
||||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
|
||||||
|
|
||||||
import org.apache.commons.lang3.tuple.Pair; |
|
||||||
|
|
||||||
import java.util.ArrayList; |
|
||||||
import java.util.HashMap; |
|
||||||
import java.util.List; |
|
||||||
import java.util.Map; |
|
||||||
|
|
||||||
import org.junit.Assert; |
|
||||||
import org.junit.Test; |
|
||||||
import org.junit.runner.RunWith; |
|
||||||
import org.mockito.Mock; |
|
||||||
import org.powermock.modules.junit4.PowerMockRunner; |
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
@RunWith(PowerMockRunner.class) |
|
||||||
public class TaskExecuteThreadTest { |
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(WorkerRegistryClientTest.class); |
|
||||||
|
|
||||||
@Mock |
|
||||||
private TaskExecutionContext taskExecutionContext; |
|
||||||
|
|
||||||
@Mock |
|
||||||
private WorkerMessageSender workerMessageSender; |
|
||||||
|
|
||||||
@Mock |
|
||||||
private AlertClientService alertClientService; |
|
||||||
|
|
||||||
@Mock |
|
||||||
private StorageOperate storageOperate; |
|
||||||
|
|
||||||
@Mock |
|
||||||
private TaskPluginManager taskPluginManager; |
|
||||||
|
|
||||||
@Test |
|
||||||
public void checkTest() { |
|
||||||
TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, |
|
||||||
"127.0.0.1:5678", |
|
||||||
workerMessageSender, |
|
||||||
alertClientService, |
|
||||||
taskPluginManager, |
|
||||||
storageOperate); |
|
||||||
|
|
||||||
String path = "/"; |
|
||||||
Map<String, String> projectRes = new HashMap<>(); |
|
||||||
projectRes.put("shell", "shell.sh"); |
|
||||||
List<Pair<String, String>> downloads = new ArrayList<>(); |
|
||||||
try { |
|
||||||
downloads = taskExecuteThread.downloadCheck(path, projectRes); |
|
||||||
} catch (Exception e) { |
|
||||||
Assert.assertNotNull(e); |
|
||||||
} |
|
||||||
downloads.add(Pair.of("shell", "shell.sh")); |
|
||||||
try{ |
|
||||||
taskExecuteThread.downloadResource(path, LOGGER, downloads); |
|
||||||
}catch (Exception e){ |
|
||||||
|
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
Loading…
Reference in new issue