Browse Source
* 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modifypull/2/head
qiaozhanwei
5 years ago
committed by
GitHub
44 changed files with 886 additions and 1413 deletions
@ -1 +1 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task request command
*/
public class ExecuteTaskAckCommand implements Serializable {
private int taskInstanceId;
private Date startTime;
private String host;
private int status;
private String logPath;
private String executePath;
public Date getStartTime() {
return startTime;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_ACK);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "ExecuteTaskAckCommand{" +
"taskInstanceId=" + taskInstanceId +
", startTime=" + startTime +
", host='" + host + '\'' +
", status=" + status +
", logPath='" + logPath + '\'' +
", executePath='" + executePath + '\'' +
'}';
}
} |
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task request command
*/
public class ExecuteTaskAckCommand implements Serializable {
/**
* taskInstanceId
*/
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
private Date startTime;
kd">private int taskInstanceId;
private String host;
*/
kd">private int taskInstanceId;
private int status;
kd">private int taskInstanceId;
/**
kd">private int taskInstanceId;
private String logPath;
*/
kd">private int taskInstanceId;
/**
* status
*/
private int status;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
private String executePath;
public Date getStartTime() {
return startTime;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_ACK);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "ExecuteTaskAckCommand{" +
"taskInstanceId=" + taskInstanceId +
", startTime=" + startTime +
", host='" + host + '\'' +
", status=" + status +
", logPath='" + logPath + '\'' +
", executePath='" + executePath + '\'' +
'}';
}
} |
||||
@ -1 +1 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task response command
*/
public class ExecuteTaskResponseCommand implements Serializable {
public ExecuteTaskResponseCommand() {
}
public ExecuteTaskResponseCommand(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
/**
* task instance id
*/
private int taskInstanceId;
/**
* status
*/
private int status;
/**
* end time
*/
private Date endTime;
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
return taskInstanceId;
}
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
kd">public int getTaskInstanceId() {
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
public int getStatus() {
kd">public int getTaskInstanceId() {
return status;
kd">public int getTaskInstanceId() {
public void setStatus(int status) {
return taskInstanceId;
return taskInstanceId;
kd">public int getTaskInstanceId() {
byte[] body = FastJsonSerializer.serialize(this);
return taskInstanceId;
}
return taskInstanceId;
}
return taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
return taskInstanceId;
public int getStatus() {
return taskInstanceId;
return status;
return taskInstanceId;
public void setStatus(int status) {
}
+
'}';
}
} |
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task response command
*/
public class ExecuteTaskResponseCommand implements Serializable {
public ExecuteTaskResponseCommand() {
}
public ExecuteTaskResponseCommand(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
/**
* task instance id
*/
private int taskInstanceId;
/**
* status
*/
private int status;
/**
* end time
*/
private Date endTime;
}
kd">public int getTaskInstanceId() {
}
return taskInstanceId;
}
}
}
}
public void setTaskInstanceId(int taskInstanceId) {
}
this.taskInstanceId = taskInstanceId;
}
public int getStatus() {
*/
}
return status;
}
public void setTaskInstanceId(int taskInstanceId) {
}
public void setStatus(int status) {
}
}
public void setTaskInstanceId(int taskInstanceId) {
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
public int getStatus() {
return status;
kd">public int getTaskInstanceId() {
public void setStatus(int status) {
this.status = status;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
return status;
return endTime;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public void setEndTime(Date endTime) {
public void setTaskInstanceId(int taskInstanceId) {
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public int getProcessId() {
return processId;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public void setProcessId(int processId) {
this.processId = processId;
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
public int getStatus() {
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
return status;
this.appIds = appIds;
kd">public int getTaskInstanceId() {
/**
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
}
}
this.taskInstanceId = taskInstanceId;
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
}
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
public int getStatus() {
}
this.taskInstanceId = taskInstanceId;
return status;
public String toString() {
return "ExecuteTaskResponseCommand{" +
"taskInstanceId=" + taskInstanceId +
public int getStatus() {
return taskInstanceId;
public int getStatus() {
}
", processId=" + processId +
", appIds='" + appIds + '\'' +
'}';
}
} |
||||
@ -1 +1 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
private int taskInstanceId;
private int processId;
private String host;
private String tenantCode;
private String logPath;
private String executePath;
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
}
public void setTenantCode(String tenantCode) {
this.tenantCode = tenantCode;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_REQUEST);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
} |
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* taskInstanceId
*/
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
private int processId;
kd">private int taskInstanceId;
private String host;
*/
kd">private int taskInstanceId;
private String tenantCode;
kd">private int taskInstanceId;
/**
kd">private int taskInstanceId;
private String logPath;
*/
kd">private int taskInstanceId;
/**
* tenantCode
*/
private String tenantCode;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
private String executePath;
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
}
public void setTenantCode(String tenantCode) {
this.tenantCode = tenantCode;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_REQUEST);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
} |
||||
@ -0,0 +1 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
/**
* kill task response command
*/
public class KillTaskResponseCommand implements Serializable {
/**
* taskInstanceId
*/
private int taskInstanceId;
/**
* host
*/
private String host;
/**
* status
*/
private int status;
/**
* processId
*/
private int processId;
/**
* other resource manager appId , for example : YARN etc
*/
protected List<String> appIds;
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public List<String> getAppIds() {
return appIds;
}
public void setAppIds(List<String> appIds) {
this.appIds = appIds;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_RESPONSE);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "KillTaskResponseCommand{" +
"taskInstanceId=" + taskInstanceId +
", host='" + host + '\'' +
", status=" + status +
", processId=" + processId +
", appIds=" + appIds +
'}';
}
} |
||||
@ -0,0 +1,48 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.cache; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.remote.entity.TaskExecutionContext; |
||||
|
||||
/** |
||||
* TaskExecutionContextCacheManager |
||||
*/ |
||||
public interface TaskExecutionContextCacheManager { |
||||
|
||||
/** |
||||
* get taskInstance by taskInstance id |
||||
* |
||||
* @param taskInstanceId taskInstanceId |
||||
* @return taskInstance |
||||
*/ |
||||
TaskExecutionContext getByTaskInstanceId(Integer taskInstanceId); |
||||
|
||||
/** |
||||
* cache taskInstance |
||||
* |
||||
* @param taskExecutionContext taskExecutionContext |
||||
*/ |
||||
void cacheTaskExecutionContext(TaskExecutionContext taskExecutionContext); |
||||
|
||||
/** |
||||
* remove taskInstance by taskInstanceId |
||||
* @param taskInstanceId taskInstanceId |
||||
*/ |
||||
void removeByTaskInstanceId(Integer taskInstanceId); |
||||
} |
||||
@ -0,0 +1,66 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.cache.impl; |
||||
|
||||
import org.apache.dolphinscheduler.remote.entity.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; |
||||
|
||||
import java.util.Map; |
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
|
||||
/** |
||||
* TaskExecutionContextCache |
||||
*/ |
||||
public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContextCacheManager { |
||||
|
||||
|
||||
/** |
||||
* taskInstance caceh |
||||
*/ |
||||
private Map<Integer,TaskExecutionContext> taskExecutionContextCache = new ConcurrentHashMap<>(); |
||||
|
||||
/** |
||||
* get taskInstance by taskInstance id |
||||
* |
||||
* @param taskInstanceId taskInstanceId |
||||
* @return taskInstance |
||||
*/ |
||||
@Override |
||||
public TaskExecutionContext getByTaskInstanceId(Integer taskInstanceId) { |
||||
return taskExecutionContextCache.get(taskInstanceId); |
||||
} |
||||
|
||||
/** |
||||
* cache taskInstance |
||||
* |
||||
* @param taskExecutionContext taskExecutionContext |
||||
*/ |
||||
@Override |
||||
public void cacheTaskExecutionContext(TaskExecutionContext taskExecutionContext) { |
||||
taskExecutionContextCache.put(taskExecutionContext.getTaskInstanceId(),taskExecutionContext); |
||||
} |
||||
|
||||
/** |
||||
* remove taskInstance by taskInstanceId |
||||
* @param taskInstanceId taskInstanceId |
||||
*/ |
||||
@Override |
||||
public void removeByTaskInstanceId(Integer taskInstanceId) { |
||||
taskExecutionContextCache.remove(taskInstanceId); |
||||
} |
||||
} |
||||
@ -0,0 +1,116 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.processor; |
||||
|
||||
|
||||
import io.netty.channel.Channel; |
||||
import io.netty.channel.ChannelFuture; |
||||
import io.netty.channel.ChannelFutureListener; |
||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
||||
import org.apache.dolphinscheduler.remote.command.ExecuteTaskAckCommand; |
||||
import org.apache.dolphinscheduler.remote.command.ExecuteTaskResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.command.KillTaskResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
|
||||
/** |
||||
* taks callback service |
||||
*/ |
||||
public class KillTaskCallbackService { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(KillTaskCallbackService.class); |
||||
|
||||
/** |
||||
* remote channels |
||||
*/ |
||||
private static final ConcurrentHashMap<Integer, NettyRemoteChannel> REMOTE_CHANNELS = new ConcurrentHashMap<>(); |
||||
|
||||
/** |
||||
* netty remoting client |
||||
*/ |
||||
private final NettyRemotingClient nettyRemotingClient; |
||||
|
||||
|
||||
public KillTaskCallbackService(){ |
||||
final NettyClientConfig clientConfig = new NettyClientConfig(); |
||||
this.nettyRemotingClient = new NettyRemotingClient(clientConfig); |
||||
} |
||||
|
||||
/** |
||||
* add callback channel |
||||
* @param taskInstanceId taskInstanceId |
||||
* @param channel channel |
||||
*/ |
||||
public void addRemoteChannel(int taskInstanceId, NettyRemoteChannel channel){ |
||||
REMOTE_CHANNELS.put(taskInstanceId, channel); |
||||
} |
||||
|
||||
/** |
||||
* get callback channel |
||||
* @param taskInstanceId taskInstanceId |
||||
* @return callback channel |
||||
*/ |
||||
public NettyRemoteChannel getRemoteChannel(int taskInstanceId){ |
||||
NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(taskInstanceId); |
||||
if(nettyRemoteChannel.isActive()){ |
||||
return nettyRemoteChannel; |
||||
} |
||||
Channel newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost()); |
||||
if(newChannel != null){ |
||||
NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel, nettyRemoteChannel.getOpaque()); |
||||
addRemoteChannel(taskInstanceId, remoteChannel); |
||||
return remoteChannel; |
||||
} |
||||
return null; |
||||
} |
||||
|
||||
/** |
||||
* remove callback channels |
||||
* @param taskInstanceId taskInstanceId |
||||
*/ |
||||
public void remove(int taskInstanceId){ |
||||
REMOTE_CHANNELS.remove(taskInstanceId); |
||||
} |
||||
|
||||
/** |
||||
* send result |
||||
* |
||||
* @param taskInstanceId taskInstanceId |
||||
* @param killTaskResponseCommand killTaskResponseCommand |
||||
*/ |
||||
public void sendKillResult(int taskInstanceId, KillTaskResponseCommand killTaskResponseCommand){ |
||||
NettyRemoteChannel nettyRemoteChannel = getRemoteChannel(taskInstanceId); |
||||
if(nettyRemoteChannel == null){ |
||||
//TODO
|
||||
} else{ |
||||
nettyRemoteChannel.writeAndFlush(killTaskResponseCommand.convert2Command()).addListener(new ChannelFutureListener(){ |
||||
|
||||
@Override |
||||
public void operationComplete(ChannelFuture future) throws Exception { |
||||
if(future.isSuccess()){ |
||||
remove(taskInstanceId); |
||||
return; |
||||
} |
||||
} |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
@ -1,365 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.worker.runner; |
||||
|
||||
import org.apache.curator.framework.recipes.locks.InterProcessMutex; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils; |
||||
import org.apache.dolphinscheduler.common.utils.*; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.Tenant; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; |
||||
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
||||
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.apache.dolphinscheduler.service.queue.ITaskQueue; |
||||
import org.apache.dolphinscheduler.service.zk.AbstractZKClient; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.Arrays; |
||||
import java.util.Date; |
||||
import java.util.List; |
||||
import java.util.concurrent.ExecutorService; |
||||
import java.util.concurrent.ThreadPoolExecutor; |
||||
|
||||
/** |
||||
* fetch task thread |
||||
*/ |
||||
public class FetchTaskThread implements Runnable{ |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FetchTaskThread.class); |
||||
/** |
||||
* set worker concurrent tasks |
||||
*/ |
||||
private final int taskNum; |
||||
|
||||
/** |
||||
* zkWorkerClient |
||||
*/ |
||||
private final ZKWorkerClient zkWorkerClient; |
||||
|
||||
/** |
||||
* task queue impl |
||||
*/ |
||||
protected ITaskQueue taskQueue; |
||||
|
||||
/** |
||||
* process database access |
||||
*/ |
||||
private final ProcessService processService; |
||||
|
||||
/** |
||||
* worker thread pool executor |
||||
*/ |
||||
private final ExecutorService workerExecService; |
||||
|
||||
/** |
||||
* worker exec nums |
||||
*/ |
||||
private int workerExecNums; |
||||
|
||||
/** |
||||
* task instance |
||||
*/ |
||||
private TaskInstance taskInstance; |
||||
|
||||
/** |
||||
* task instance id |
||||
*/ |
||||
Integer taskInstId; |
||||
|
||||
/** |
||||
* worker config |
||||
*/ |
||||
private WorkerConfig workerConfig; |
||||
|
||||
public FetchTaskThread(ZKWorkerClient zkWorkerClient, |
||||
ProcessService processService, |
||||
ITaskQueue taskQueue){ |
||||
this.zkWorkerClient = zkWorkerClient; |
||||
this.processService = processService; |
||||
this.taskQueue = taskQueue; |
||||
this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class); |
||||
this.taskNum = workerConfig.getWorkerFetchTaskNum(); |
||||
this.workerExecNums = workerConfig.getWorkerExecThreads(); |
||||
// worker thread pool executor
|
||||
this.workerExecService = ThreadUtils.newDaemonFixedThreadExecutor("Worker-Fetch-Task-Thread", workerExecNums); |
||||
this.taskInstance = null; |
||||
} |
||||
|
||||
/** |
||||
* Check if the task runs on this worker |
||||
* @param taskInstance |
||||
* @param host |
||||
* @return |
||||
*/ |
||||
private boolean checkWorkerGroup(TaskInstance taskInstance, String host){ |
||||
|
||||
int taskWorkerGroupId = processService.getTaskWorkerGroupId(taskInstance); |
||||
|
||||
if(taskWorkerGroupId <= 0){ |
||||
return true; |
||||
} |
||||
WorkerGroup workerGroup = processService.queryWorkerGroupById(taskWorkerGroupId); |
||||
if(workerGroup == null ){ |
||||
logger.info("task {} cannot find the worker group, use all worker instead.", taskInstance.getId()); |
||||
return true; |
||||
} |
||||
String ips = workerGroup.getIpList(); |
||||
if(StringUtils.isBlank(ips)){ |
||||
logger.error("task:{} worker group:{} parameters(ip_list) is null, this task would be running on all workers", |
||||
taskInstance.getId(), workerGroup.getId()); |
||||
} |
||||
String[] ipArray = ips.split(Constants.COMMA); |
||||
List<String> ipList = Arrays.asList(ipArray); |
||||
return ipList.contains(host); |
||||
} |
||||
|
||||
|
||||
|
||||
|
||||
@Override |
||||
public void run() { |
||||
logger.info("worker start fetch tasks..."); |
||||
while (Stopper.isRunning()){ |
||||
InterProcessMutex mutex = null; |
||||
String currentTaskQueueStr = null; |
||||
try { |
||||
ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) workerExecService; |
||||
//check memory and cpu usage and threads
|
||||
boolean runCheckFlag = OSUtils.checkResource(workerConfig.getWorkerMaxCpuloadAvg(), workerConfig.getWorkerReservedMemory()) && checkThreadCount(poolExecutor); |
||||
|
||||
if(!runCheckFlag) { |
||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
||||
continue; |
||||
} |
||||
|
||||
//whether have tasks, if no tasks , no need lock //get all tasks
|
||||
boolean hasTask = taskQueue.hasTask(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); |
||||
|
||||
if (!hasTask){ |
||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
||||
continue; |
||||
} |
||||
// creating distributed locks, lock path /dolphinscheduler/lock/worker
|
||||
mutex = zkWorkerClient.acquireZkLock(zkWorkerClient.getZkClient(), |
||||
zkWorkerClient.getWorkerLockPath()); |
||||
|
||||
|
||||
// task instance id str
|
||||
List<String> taskQueueStrArr = taskQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskNum); |
||||
|
||||
for(String taskQueueStr : taskQueueStrArr){ |
||||
|
||||
currentTaskQueueStr = taskQueueStr; |
||||
|
||||
if (StringUtils.isEmpty(taskQueueStr)) { |
||||
continue; |
||||
} |
||||
|
||||
if (!checkThreadCount(poolExecutor)) { |
||||
break; |
||||
} |
||||
|
||||
// get task instance id
|
||||
taskInstId = getTaskInstanceId(taskQueueStr); |
||||
|
||||
// mainly to wait for the master insert task to succeed
|
||||
waitForTaskInstance(); |
||||
|
||||
taskInstance = processService.getTaskInstanceDetailByTaskId(taskInstId); |
||||
|
||||
// verify task instance is null
|
||||
if (verifyTaskInstanceIsNull(taskInstance)) { |
||||
logger.warn("remove task queue : {} due to taskInstance is null", taskQueueStr); |
||||
processErrorTask(taskQueueStr); |
||||
continue; |
||||
} |
||||
|
||||
if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ |
||||
continue; |
||||
} |
||||
|
||||
// if process definition is null ,process definition already deleted
|
||||
int userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId(); |
||||
|
||||
Tenant tenant = processService.getTenantForProcess( |
||||
taskInstance.getProcessInstance().getTenantId(), |
||||
userId); |
||||
|
||||
// verify tenant is null
|
||||
if (verifyTenantIsNull(tenant)) { |
||||
logger.warn("remove task queue : {} due to tenant is null", taskQueueStr); |
||||
processErrorTask(taskQueueStr); |
||||
continue; |
||||
} |
||||
|
||||
// set queue for process instance, user-specified queue takes precedence over tenant queue
|
||||
String userQueue = processService.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId()); |
||||
taskInstance.getProcessInstance().setQueue(StringUtils.isEmpty(userQueue) ? tenant.getQueue() : userQueue); |
||||
taskInstance.getProcessInstance().setTenantCode(tenant.getTenantCode()); |
||||
|
||||
logger.info("worker fetch taskId : {} from queue ", taskInstId); |
||||
|
||||
// local execute path
|
||||
String execLocalPath = getExecLocalPath(); |
||||
|
||||
logger.info("task instance local execute path : {} ", execLocalPath); |
||||
|
||||
// init task
|
||||
taskInstance.init(OSUtils.getHost(), |
||||
new Date(), |
||||
execLocalPath); |
||||
|
||||
// check and create users
|
||||
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, |
||||
tenant.getTenantCode()); |
||||
|
||||
logger.info("task : {} ready to submit to task scheduler thread",taskInstId); |
||||
// submit task
|
||||
// workerExecService.submit(new TaskExecuteThread(taskInstance, processService));
|
||||
|
||||
// remove node from zk
|
||||
removeNodeFromTaskQueue(taskQueueStr); |
||||
} |
||||
|
||||
}catch (Exception e){ |
||||
processErrorTask(currentTaskQueueStr); |
||||
logger.error("fetch task thread failure" ,e); |
||||
}finally { |
||||
AbstractZKClient.releaseMutex(mutex); |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* process error task |
||||
* |
||||
* @param taskQueueStr task queue str |
||||
*/ |
||||
private void processErrorTask(String taskQueueStr){ |
||||
// remove from zk
|
||||
removeNodeFromTaskQueue(taskQueueStr); |
||||
|
||||
if (taskInstance != null){ |
||||
processService.changeTaskState(ExecutionStatus.FAILURE, |
||||
taskInstance.getStartTime(), |
||||
taskInstance.getHost(), |
||||
null, |
||||
null, |
||||
taskInstId); |
||||
} |
||||
|
||||
} |
||||
|
||||
/** |
||||
* remove node from task queue |
||||
* |
||||
* @param taskQueueStr task queue |
||||
*/ |
||||
private void removeNodeFromTaskQueue(String taskQueueStr){ |
||||
taskQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskQueueStr); |
||||
} |
||||
|
||||
/** |
||||
* verify task instance is null |
||||
* @param taskInstance |
||||
* @return true if task instance is null |
||||
*/ |
||||
private boolean verifyTaskInstanceIsNull(TaskInstance taskInstance) { |
||||
if (taskInstance == null ) { |
||||
logger.error("task instance is null. task id : {} ", taskInstId); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* verify tenant is null |
||||
* |
||||
* @param tenant tenant |
||||
* @return true if tenant is null |
||||
*/ |
||||
private boolean verifyTenantIsNull(Tenant tenant) { |
||||
if(tenant == null){ |
||||
logger.error("tenant not exists,process instance id : {},task instance id : {}", |
||||
taskInstance.getProcessInstance().getId(), |
||||
taskInstance.getId()); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* get execute local path |
||||
* |
||||
* @return execute local path |
||||
*/ |
||||
private String getExecLocalPath(){ |
||||
return FileUtils.getProcessExecDir(taskInstance.getProcessDefine().getProjectId(), |
||||
taskInstance.getProcessDefine().getId(), |
||||
taskInstance.getProcessInstance().getId(), |
||||
taskInstance.getId()); |
||||
} |
||||
|
||||
/** |
||||
* check thread count |
||||
* |
||||
* @param poolExecutor pool executor |
||||
* @return true if active count < worker exec nums |
||||
*/ |
||||
private boolean checkThreadCount(ThreadPoolExecutor poolExecutor) { |
||||
int activeCount = poolExecutor.getActiveCount(); |
||||
if (activeCount >= workerExecNums) { |
||||
logger.info("thread insufficient , activeCount : {} , " + |
||||
"workerExecNums : {}, will sleep : {} millis for thread resource", |
||||
activeCount, |
||||
workerExecNums, |
||||
Constants.SLEEP_TIME_MILLIS); |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
/** |
||||
* wait for task instance exists, because of db action would be delayed. |
||||
* |
||||
* @throws Exception exception |
||||
*/ |
||||
private void waitForTaskInstance()throws Exception{ |
||||
int retryTimes = 30; |
||||
while (taskInstance == null && retryTimes > 0) { |
||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
||||
taskInstance = processService.findTaskInstanceById(taskInstId); |
||||
retryTimes--; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get task instance id |
||||
* |
||||
* @param taskQueueStr task queue |
||||
* @return task instance id |
||||
*/ |
||||
private int getTaskInstanceId(String taskQueueStr){ |
||||
return Integer.parseInt(taskQueueStr.split(Constants.UNDERLINE)[3]); |
||||
} |
||||
} |
||||
@ -1,237 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.worker.task.dependent; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
||||
import org.apache.dolphinscheduler.common.enums.DependentRelation; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.model.DateInterval; |
||||
import org.apache.dolphinscheduler.common.model.DependentItem; |
||||
import org.apache.dolphinscheduler.common.utils.DependentUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.*; |
||||
|
||||
/** |
||||
* dependent item execute |
||||
*/ |
||||
public class DependentExecute { |
||||
/** |
||||
* process service |
||||
*/ |
||||
private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class); |
||||
|
||||
/** |
||||
* depend item list |
||||
*/ |
||||
private List<DependentItem> dependItemList; |
||||
|
||||
/** |
||||
* dependent relation |
||||
*/ |
||||
private DependentRelation relation; |
||||
|
||||
/** |
||||
* depend result |
||||
*/ |
||||
private DependResult modelDependResult = DependResult.WAITING; |
||||
|
||||
/** |
||||
* depend result map |
||||
*/ |
||||
private Map<String, DependResult> dependResultMap = new HashMap<>(); |
||||
|
||||
/** |
||||
* logger |
||||
*/ |
||||
private Logger logger = LoggerFactory.getLogger(DependentExecute.class); |
||||
|
||||
/** |
||||
* constructor |
||||
* @param itemList item list |
||||
* @param relation relation |
||||
*/ |
||||
public DependentExecute(List<DependentItem> itemList, DependentRelation relation){ |
||||
this.dependItemList = itemList; |
||||
this.relation = relation; |
||||
} |
||||
|
||||
/** |
||||
* get dependent item for one dependent item |
||||
* @param dependentItem dependent item |
||||
* @param currentTime current time |
||||
* @return DependResult |
||||
*/ |
||||
public DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){ |
||||
List<DateInterval> dateIntervals = DependentUtils.getDateIntervalList(currentTime, dependentItem.getDateValue()); |
||||
return calculateResultForTasks(dependentItem, dateIntervals ); |
||||
} |
||||
|
||||
/** |
||||
* calculate dependent result for one dependent item. |
||||
* @param dependentItem dependent item |
||||
* @param dateIntervals date intervals |
||||
* @return dateIntervals |
||||
*/ |
||||
private DependResult calculateResultForTasks(DependentItem dependentItem, |
||||
List<DateInterval> dateIntervals) { |
||||
DependResult result = DependResult.FAILED; |
||||
for(DateInterval dateInterval : dateIntervals){ |
||||
ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(), |
||||
dateInterval); |
||||
if(processInstance == null){ |
||||
logger.error("cannot find the right process instance: definition id:{}, start:{}, end:{}", |
||||
dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() ); |
||||
return DependResult.FAILED; |
||||
} |
||||
if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){ |
||||
result = getDependResultByState(processInstance.getState()); |
||||
}else{ |
||||
TaskInstance taskInstance = null; |
||||
List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); |
||||
|
||||
for(TaskInstance task : taskInstanceList){ |
||||
if(task.getName().equals(dependentItem.getDepTasks())){ |
||||
taskInstance = task; |
||||
break; |
||||
} |
||||
} |
||||
if(taskInstance == null){ |
||||
// cannot find task in the process instance
|
||||
// maybe because process instance is running or failed.
|
||||
result = getDependResultByState(processInstance.getState()); |
||||
}else{ |
||||
result = getDependResultByState(taskInstance.getState()); |
||||
} |
||||
} |
||||
if(result != DependResult.SUCCESS){ |
||||
break; |
||||
} |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* find the last one process instance that : |
||||
* 1. manual run and finish between the interval |
||||
* 2. schedule run and schedule time between the interval |
||||
* @param definitionId definition id |
||||
* @param dateInterval date interval |
||||
* @return ProcessInstance |
||||
*/ |
||||
private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) { |
||||
|
||||
ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval); |
||||
if(runningProcess != null){ |
||||
return runningProcess; |
||||
} |
||||
|
||||
ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval( |
||||
definitionId, dateInterval |
||||
); |
||||
|
||||
ProcessInstance lastManualProcess = processService.findLastManualProcessInterval( |
||||
definitionId, dateInterval |
||||
); |
||||
|
||||
if(lastManualProcess ==null){ |
||||
return lastSchedulerProcess; |
||||
} |
||||
if(lastSchedulerProcess == null){ |
||||
return lastManualProcess; |
||||
} |
||||
|
||||
return (lastManualProcess.getEndTime().after(lastSchedulerProcess.getEndTime()))? |
||||
lastManualProcess : lastSchedulerProcess; |
||||
} |
||||
|
||||
/** |
||||
* get dependent result by task/process instance state |
||||
* @param state state |
||||
* @return DependResult |
||||
*/ |
||||
private DependResult getDependResultByState(ExecutionStatus state) { |
||||
|
||||
if(state.typeIsRunning() || state == ExecutionStatus.SUBMITTED_SUCCESS || state == ExecutionStatus.WAITTING_THREAD){ |
||||
return DependResult.WAITING; |
||||
}else if(state.typeIsSuccess()){ |
||||
return DependResult.SUCCESS; |
||||
}else{ |
||||
return DependResult.FAILED; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* judge depend item finished |
||||
* @param currentTime current time |
||||
* @return boolean |
||||
*/ |
||||
public boolean finish(Date currentTime){ |
||||
if(modelDependResult == DependResult.WAITING){ |
||||
modelDependResult = getModelDependResult(currentTime); |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
/** |
||||
* get model depend result |
||||
* @param currentTime current time |
||||
* @return DependResult |
||||
*/ |
||||
public DependResult getModelDependResult(Date currentTime){ |
||||
|
||||
List<DependResult> dependResultList = new ArrayList<>(); |
||||
|
||||
for(DependentItem dependentItem : dependItemList){ |
||||
DependResult dependResult = getDependResultForItem(dependentItem, currentTime); |
||||
if(dependResult != DependResult.WAITING){ |
||||
dependResultMap.put(dependentItem.getKey(), dependResult); |
||||
} |
||||
dependResultList.add(dependResult); |
||||
} |
||||
modelDependResult = DependentUtils.getDependResultForRelation( |
||||
this.relation, dependResultList |
||||
); |
||||
return modelDependResult; |
||||
} |
||||
|
||||
/** |
||||
* get dependent item result |
||||
* @param item item |
||||
* @param currentTime current time |
||||
* @return DependResult |
||||
*/ |
||||
public DependResult getDependResultForItem(DependentItem item, Date currentTime){ |
||||
String key = item.getKey(); |
||||
if(dependResultMap.containsKey(key)){ |
||||
return dependResultMap.get(key); |
||||
} |
||||
return getDependentResultForItem(item, currentTime); |
||||
} |
||||
|
||||
public Map<String, DependResult> getDependResultMap(){ |
||||
return dependResultMap; |
||||
} |
||||
|
||||
} |
||||
@ -1,191 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.worker.task.dependent; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.model.DependentTaskModel; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; |
||||
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||
import org.apache.dolphinscheduler.common.utils.DependentUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.server.worker.task.AbstractTask; |
||||
import org.apache.dolphinscheduler.server.worker.task.TaskProps; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.slf4j.Logger; |
||||
|
||||
import java.util.*; |
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; |
||||
|
||||
/** |
||||
* Dependent Task |
||||
*/ |
||||
public class DependentTask extends AbstractTask { |
||||
|
||||
/** |
||||
* dependent task list |
||||
*/ |
||||
private List<DependentExecute> dependentTaskList = new ArrayList<>(); |
||||
|
||||
/** |
||||
* depend item result map |
||||
* save the result to log file |
||||
*/ |
||||
private Map<String, DependResult> dependResultMap = new HashMap<>(); |
||||
|
||||
/** |
||||
* dependent parameters |
||||
*/ |
||||
private DependentParameters dependentParameters; |
||||
|
||||
/** |
||||
* dependent date |
||||
*/ |
||||
private Date dependentDate; |
||||
|
||||
/** |
||||
* process service |
||||
*/ |
||||
private ProcessService processService; |
||||
|
||||
/** |
||||
* constructor |
||||
* @param props props |
||||
* @param logger logger |
||||
*/ |
||||
public DependentTask(TaskProps props, Logger logger) { |
||||
super(props, logger); |
||||
} |
||||
|
||||
@Override |
||||
public void init(){ |
||||
logger.info("dependent task initialize"); |
||||
|
||||
this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(), |
||||
DependentParameters.class); |
||||
|
||||
for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){ |
||||
this.dependentTaskList.add(new DependentExecute( |
||||
taskModel.getDependItemList(), taskModel.getRelation())); |
||||
} |
||||
|
||||
this.processService = SpringApplicationContext.getBean(ProcessService.class); |
||||
|
||||
if(taskProps.getScheduleTime() != null){ |
||||
this.dependentDate = taskProps.getScheduleTime(); |
||||
}else{ |
||||
this.dependentDate = taskProps.getTaskStartTime(); |
||||
} |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public void handle() throws Exception { |
||||
// set the name of the current thread
|
||||
String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); |
||||
Thread.currentThread().setName(threadLoggerInfoName); |
||||
|
||||
try{ |
||||
TaskInstance taskInstance = null; |
||||
while(Stopper.isRunning()){ |
||||
taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstanceId()); |
||||
|
||||
if(taskInstance == null){ |
||||
exitStatusCode = -1; |
||||
break; |
||||
} |
||||
|
||||
if(taskInstance.getState() == ExecutionStatus.KILL){ |
||||
this.cancel = true; |
||||
} |
||||
|
||||
if(this.cancel || allDependentTaskFinish()){ |
||||
break; |
||||
} |
||||
|
||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
||||
} |
||||
|
||||
if(cancel){ |
||||
exitStatusCode = Constants.EXIT_CODE_KILL; |
||||
}else{ |
||||
DependResult result = getTaskDependResult(); |
||||
exitStatusCode = (result == DependResult.SUCCESS) ? |
||||
Constants.EXIT_CODE_SUCCESS : Constants.EXIT_CODE_FAILURE; |
||||
} |
||||
}catch (Exception e){ |
||||
logger.error(e.getMessage(),e); |
||||
exitStatusCode = -1; |
||||
throw e; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get dependent result |
||||
* @return DependResult |
||||
*/ |
||||
private DependResult getTaskDependResult(){ |
||||
List<DependResult> dependResultList = new ArrayList<>(); |
||||
for(DependentExecute dependentExecute : dependentTaskList){ |
||||
DependResult dependResult = dependentExecute.getModelDependResult(dependentDate); |
||||
dependResultList.add(dependResult); |
||||
} |
||||
DependResult result = DependentUtils.getDependResultForRelation( |
||||
this.dependentParameters.getRelation(), dependResultList |
||||
); |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* judge all dependent tasks finish |
||||
* @return whether all dependent tasks finish |
||||
*/ |
||||
private boolean allDependentTaskFinish(){ |
||||
boolean finish = true; |
||||
for(DependentExecute dependentExecute : dependentTaskList){ |
||||
for(Map.Entry<String, DependResult> entry: dependentExecute.getDependResultMap().entrySet()) { |
||||
if(!dependResultMap.containsKey(entry.getKey())){ |
||||
dependResultMap.put(entry.getKey(), entry.getValue()); |
||||
//save depend result to log
|
||||
logger.info("dependent item complete {} {},{}", |
||||
DEPENDENT_SPLIT, entry.getKey(), entry.getValue().toString()); |
||||
} |
||||
} |
||||
if(!dependentExecute.finish(dependentDate)){ |
||||
finish = false; |
||||
} |
||||
} |
||||
return finish; |
||||
} |
||||
|
||||
|
||||
@Override |
||||
public void cancelApplication(boolean cancelApplication) throws Exception { |
||||
// cancel process
|
||||
this.cancel = true; |
||||
} |
||||
|
||||
@Override |
||||
public AbstractParameters getParameters() { |
||||
return null; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue