Browse Source
* #7698 * #7698 * #7698 * fix workflow is always running * fix workflow is always running * fix workflow is always running * #7700 #7698 * add license * fix ut * #7700 #7698 Co-authored-by: honghuo.zw <honghuo.zw@alibaba-inc.com>2.0.7-release
zwZjut
3 years ago
committed by
GitHub
24 changed files with 483 additions and 33 deletions
@ -0,0 +1,102 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
import java.util.concurrent.ThreadPoolExecutor; |
||||
import java.util.concurrent.TimeUnit; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import com.google.common.util.concurrent.FutureCallback; |
||||
import com.google.common.util.concurrent.Futures; |
||||
import com.google.common.util.concurrent.ListenableFuture; |
||||
import com.google.common.util.concurrent.ListeningExecutorService; |
||||
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
||||
public class MasterExecService { |
||||
|
||||
/** |
||||
* logger of MasterExecService |
||||
*/ |
||||
private static final Logger logger = LoggerFactory.getLogger(MasterExecService.class); |
||||
|
||||
/** |
||||
* master exec service |
||||
*/ |
||||
private final ThreadPoolExecutor execService; |
||||
|
||||
private final ListeningExecutorService listeningExecutorService; |
||||
|
||||
/** |
||||
* start process failed map |
||||
*/ |
||||
private final ConcurrentHashMap<Integer, WorkflowExecuteThread> startProcessFailedMap; |
||||
|
||||
private final ConcurrentHashMap<Integer, WorkflowExecuteThread> filterMap = new ConcurrentHashMap<>(); |
||||
|
||||
public MasterExecService(ConcurrentHashMap<Integer, WorkflowExecuteThread> startProcessFailedMap,ThreadPoolExecutor execService) { |
||||
this.startProcessFailedMap = startProcessFailedMap; |
||||
this.execService = execService; |
||||
this.listeningExecutorService = MoreExecutors.listeningDecorator(this.execService); |
||||
} |
||||
|
||||
public void execute(WorkflowExecuteThread workflowExecuteThread) { |
||||
if (workflowExecuteThread == null |
||||
|| workflowExecuteThread.getProcessInstance() == null |
||||
|| workflowExecuteThread.isStart() |
||||
|| filterMap.containsKey(workflowExecuteThread.getProcessInstance().getId())) { |
||||
return; |
||||
} |
||||
Integer processInstanceId = workflowExecuteThread.getProcessInstance().getId(); |
||||
filterMap.put(processInstanceId, workflowExecuteThread); |
||||
ListenableFuture future = this.listeningExecutorService.submit(workflowExecuteThread); |
||||
FutureCallback futureCallback = new FutureCallback() { |
||||
@Override |
||||
public void onSuccess(Object o) { |
||||
if (!workflowExecuteThread.isStart()) { |
||||
startProcessFailedMap.putIfAbsent(processInstanceId, workflowExecuteThread); |
||||
} else { |
||||
startProcessFailedMap.remove(processInstanceId); |
||||
} |
||||
filterMap.remove(processInstanceId); |
||||
} |
||||
|
||||
@Override |
||||
public void onFailure(Throwable throwable) { |
||||
logger.error("handle events {} failed", processInstanceId, throwable); |
||||
if (!workflowExecuteThread.isStart()) { |
||||
startProcessFailedMap.putIfAbsent(processInstanceId, workflowExecuteThread); |
||||
} else { |
||||
startProcessFailedMap.remove(processInstanceId); |
||||
} |
||||
filterMap.remove(processInstanceId); |
||||
} |
||||
}; |
||||
Futures.addCallback(future, futureCallback, this.listeningExecutorService); |
||||
} |
||||
|
||||
public void shutdown() { |
||||
this.execService.shutdown(); |
||||
} |
||||
|
||||
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { |
||||
return this.execService.awaitTermination(timeout, unit); |
||||
} |
||||
} |
||||
@ -0,0 +1,85 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.runner; |
||||
|
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
import java.util.concurrent.ExecutorService; |
||||
import java.util.concurrent.ThreadPoolExecutor; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import com.google.common.util.concurrent.FutureCallback; |
||||
import com.google.common.util.concurrent.Futures; |
||||
import com.google.common.util.concurrent.ListenableFuture; |
||||
import com.google.common.util.concurrent.ListeningExecutorService; |
||||
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
||||
public class WorkerExecService { |
||||
/** |
||||
* logger of WorkerExecService |
||||
*/ |
||||
private static final Logger logger = LoggerFactory.getLogger(WorkerExecService.class); |
||||
|
||||
private final ListeningExecutorService listeningExecutorService; |
||||
|
||||
/** |
||||
* thread executor service |
||||
*/ |
||||
private final ExecutorService execService; |
||||
|
||||
/** |
||||
* running task |
||||
*/ |
||||
private final ConcurrentHashMap<Integer, TaskExecuteThread> taskExecuteThreadMap; |
||||
|
||||
public WorkerExecService(ExecutorService execService, ConcurrentHashMap<Integer, TaskExecuteThread> taskExecuteThreadMap) { |
||||
this.execService = execService; |
||||
this.listeningExecutorService = MoreExecutors.listeningDecorator(this.execService); |
||||
this.taskExecuteThreadMap = taskExecuteThreadMap; |
||||
} |
||||
|
||||
public void submit(TaskExecuteThread taskExecuteThread) { |
||||
taskExecuteThreadMap.put(taskExecuteThread.getTaskExecutionContext().getTaskInstanceId(), taskExecuteThread); |
||||
ListenableFuture future = this.listeningExecutorService.submit(taskExecuteThread); |
||||
FutureCallback futureCallback = new FutureCallback() { |
||||
@Override |
||||
public void onSuccess(Object o) { |
||||
taskExecuteThreadMap.remove(taskExecuteThread.getTaskExecutionContext().getTaskInstanceId()); |
||||
} |
||||
|
||||
@Override |
||||
public void onFailure(Throwable throwable) { |
||||
logger.error("task execute failed, processInstanceId:{}, taskInstanceId:{}", taskExecuteThread.getTaskExecutionContext().getProcessInstanceId() |
||||
, taskExecuteThread.getTaskExecutionContext().getTaskInstanceId(), throwable); |
||||
taskExecuteThreadMap.remove(taskExecuteThread.getTaskExecutionContext().getTaskInstanceId()); |
||||
} |
||||
}; |
||||
Futures.addCallback(future, futureCallback, this.listeningExecutorService); |
||||
} |
||||
|
||||
/** |
||||
* get thread pool queue size |
||||
* |
||||
* @return queue size |
||||
*/ |
||||
public int getThreadPoolQueueSize() { |
||||
return ((ThreadPoolExecutor) this.execService).getQueue().size(); |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue