Wenjun Ruan
5 months ago
committed by
GitHub
65 changed files with 403 additions and 579 deletions
@ -0,0 +1,55 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
|
||||
public abstract class BaseTaskExecuteRunnable implements TaskExecuteRunnable { |
||||
|
||||
protected final ProcessInstance workflowInstance; |
||||
protected final TaskInstance taskInstance; |
||||
protected final TaskExecutionContext taskExecutionContext; |
||||
|
||||
public BaseTaskExecuteRunnable(ProcessInstance workflowInstance, |
||||
TaskInstance taskInstance, |
||||
TaskExecutionContext taskExecutionContext) { |
||||
this.taskInstance = checkNotNull(taskInstance); |
||||
this.workflowInstance = checkNotNull(workflowInstance); |
||||
this.taskExecutionContext = checkNotNull(taskExecutionContext); |
||||
} |
||||
|
||||
@Override |
||||
public ProcessInstance getWorkflowInstance() { |
||||
return workflowInstance; |
||||
} |
||||
|
||||
@Override |
||||
public TaskInstance getTaskInstance() { |
||||
return taskInstance; |
||||
} |
||||
|
||||
@Override |
||||
public TaskExecutionContext getTaskExecutionContext() { |
||||
return taskExecutionContext; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,52 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutor; |
||||
|
||||
import java.util.concurrent.BlockingQueue; |
||||
import java.util.concurrent.LinkedBlockingQueue; |
||||
|
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* |
||||
*/ |
||||
@Component |
||||
public class GlobalMasterTaskExecuteRunnableQueue { |
||||
|
||||
private final BlockingQueue<MasterTaskExecutor> masterTaskExecutorBlockingQueue = |
||||
new LinkedBlockingQueue<>(); |
||||
|
||||
public boolean submitMasterTaskExecuteRunnable(MasterTaskExecutor masterTaskExecutor) { |
||||
return masterTaskExecutorBlockingQueue.offer(masterTaskExecutor); |
||||
} |
||||
|
||||
public MasterTaskExecutor takeMasterTaskExecuteRunnable() throws InterruptedException { |
||||
return masterTaskExecutorBlockingQueue.take(); |
||||
} |
||||
|
||||
public boolean removeMasterTaskExecuteRunnable(MasterTaskExecutor masterTaskExecutor) { |
||||
return masterTaskExecutorBlockingQueue.remove(masterTaskExecutor); |
||||
} |
||||
|
||||
public int size() { |
||||
return masterTaskExecutorBlockingQueue.size(); |
||||
} |
||||
|
||||
} |
||||
@ -1,54 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import org.apache.dolphinscheduler.server.master.runner.execute.MasterDelayTaskExecuteRunnable; |
||||
import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnable; |
||||
|
||||
import java.util.concurrent.DelayQueue; |
||||
|
||||
import org.springframework.stereotype.Component; |
||||
|
||||
/** |
||||
* |
||||
*/ |
||||
@Component |
||||
public class MasterDelayTaskExecuteRunnableDelayQueue { |
||||
|
||||
private final DelayQueue<MasterDelayTaskExecuteRunnable> masterDelayTaskExecuteRunnableDelayQueue = |
||||
new DelayQueue<>(); |
||||
|
||||
public boolean submitMasterDelayTaskExecuteRunnable(MasterDelayTaskExecuteRunnable masterDelayTaskExecuteRunnable) { |
||||
return masterDelayTaskExecuteRunnableDelayQueue.offer(masterDelayTaskExecuteRunnable); |
||||
} |
||||
|
||||
public MasterDelayTaskExecuteRunnable takeMasterDelayTaskExecuteRunnable() throws InterruptedException { |
||||
return masterDelayTaskExecuteRunnableDelayQueue.take(); |
||||
} |
||||
|
||||
// todo: if we move the delay process to master, than we don't need this method, since dispatchProcess can directly
|
||||
// submit to thread pool
|
||||
public boolean removeMasterDelayTaskExecuteRunnable(MasterTaskExecuteRunnable masterTaskExecuteRunnable) { |
||||
return masterDelayTaskExecuteRunnableDelayQueue.remove(masterTaskExecuteRunnable); |
||||
} |
||||
|
||||
public int size() { |
||||
return masterDelayTaskExecuteRunnableDelayQueue.size(); |
||||
} |
||||
|
||||
} |
||||
@ -1,68 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner.execute; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; |
||||
import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; |
||||
|
||||
import java.util.concurrent.Delayed; |
||||
import java.util.concurrent.TimeUnit; |
||||
|
||||
public abstract class MasterDelayTaskExecuteRunnable extends MasterTaskExecuteRunnable implements Delayed { |
||||
|
||||
public MasterDelayTaskExecuteRunnable(TaskExecutionContext taskExecutionContext, |
||||
LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder, |
||||
LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) { |
||||
super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager); |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(Object obj) { |
||||
if (!(obj instanceof MasterDelayTaskExecuteRunnable)) { |
||||
return false; |
||||
} |
||||
MasterDelayTaskExecuteRunnable other = (MasterDelayTaskExecuteRunnable) obj; |
||||
return other.getTaskExecutionContext().getTaskInstanceId() == this.getTaskExecutionContext() |
||||
.getTaskInstanceId(); |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
return this.getTaskExecutionContext().getTaskInstanceId(); |
||||
} |
||||
|
||||
@Override |
||||
public long getDelay(TimeUnit unit) { |
||||
TaskExecutionContext taskExecutionContext = getTaskExecutionContext(); |
||||
return unit.convert( |
||||
DateUtils.getRemainTime( |
||||
taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L), |
||||
TimeUnit.SECONDS); |
||||
} |
||||
|
||||
@Override |
||||
public int compareTo(Delayed o) { |
||||
if (o == null) { |
||||
return 1; |
||||
} |
||||
return Long.compare(this.getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); |
||||
} |
||||
|
||||
} |
||||
@ -1,59 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.runner; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; |
||||
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; |
||||
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||
|
||||
import javax.annotation.Nullable; |
||||
|
||||
import lombok.NonNull; |
||||
|
||||
public class DefaultWorkerDelayTaskExecuteRunnableFactory |
||||
extends |
||||
WorkerDelayTaskExecuteRunnableFactory<DefaultWorkerDelayTaskExecuteRunnable> { |
||||
|
||||
protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, |
||||
@NonNull WorkerConfig workerConfig, |
||||
@NonNull WorkerMessageSender workerMessageSender, |
||||
@NonNull TaskPluginManager taskPluginManager, |
||||
@Nullable StorageOperate storageOperate, |
||||
@NonNull WorkerRegistryClient workerRegistryClient) { |
||||
super(taskExecutionContext, |
||||
workerConfig, |
||||
workerMessageSender, |
||||
taskPluginManager, |
||||
storageOperate, |
||||
workerRegistryClient); |
||||
} |
||||
|
||||
@Override |
||||
public DefaultWorkerDelayTaskExecuteRunnable createWorkerTaskExecuteRunnable() { |
||||
return new DefaultWorkerDelayTaskExecuteRunnable( |
||||
taskExecutionContext, |
||||
workerConfig, |
||||
workerMessageSender, |
||||
taskPluginManager, |
||||
storageOperate, |
||||
workerRegistryClient); |
||||
} |
||||
} |
||||
@ -1,69 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.runner; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; |
||||
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; |
||||
import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; |
||||
|
||||
import java.util.concurrent.Delayed; |
||||
import java.util.concurrent.TimeUnit; |
||||
|
||||
import javax.annotation.Nullable; |
||||
|
||||
import lombok.NonNull; |
||||
|
||||
public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRunnable implements Delayed { |
||||
|
||||
protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, |
||||
@NonNull WorkerConfig workerConfig, |
||||
@NonNull WorkerMessageSender workerMessageSender, |
||||
@NonNull TaskPluginManager taskPluginManager, |
||||
@Nullable StorageOperate storageOperate, |
||||
@NonNull WorkerRegistryClient workerRegistryClient) { |
||||
super(taskExecutionContext, |
||||
workerConfig, |
||||
workerMessageSender, |
||||
taskPluginManager, |
||||
storageOperate, |
||||
workerRegistryClient); |
||||
} |
||||
|
||||
@Override |
||||
public long getDelay(TimeUnit unit) { |
||||
TaskExecutionContext taskExecutionContext = getTaskExecutionContext(); |
||||
return unit.convert( |
||||
DateUtils.getRemainTime( |
||||
DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), |
||||
taskExecutionContext.getDelayTime() * 60L), |
||||
TimeUnit.SECONDS); |
||||
} |
||||
|
||||
@Override |
||||
public int compareTo(Delayed o) { |
||||
if (o == null) { |
||||
return 1; |
||||
} |
||||
return Long.compare(this.getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue