Browse Source
* 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modifypull/2/head
qiaozhanwei
5 years ago
committed by
GitHub
44 changed files with 886 additions and 1413 deletions
@ -1 +1 @@ |
|||||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task request command
*/
public class ExecuteTaskAckCommand implements Serializable {
private int taskInstanceId;
private Date startTime;
private String host;
private int status;
private String logPath;
private String executePath;
public Date getStartTime() {
return startTime;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_ACK);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "ExecuteTaskAckCommand{" +
"taskInstanceId=" + taskInstanceId +
", startTime=" + startTime +
", host='" + host + '\'' +
", status=" + status +
", logPath='" + logPath + '\'' +
", executePath='" + executePath + '\'' +
'}';
}
} |
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task request command
*/
public class ExecuteTaskAckCommand implements Serializable {
/**
* taskInstanceId
*/
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
private Date startTime;
kd">private int taskInstanceId;
private String host;
*/
kd">private int taskInstanceId;
private int status;
kd">private int taskInstanceId;
/**
kd">private int taskInstanceId;
private String logPath;
*/
kd">private int taskInstanceId;
/**
* status
*/
private int status;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
private String executePath;
public Date getStartTime() {
return startTime;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_ACK);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "ExecuteTaskAckCommand{" +
"taskInstanceId=" + taskInstanceId +
", startTime=" + startTime +
", host='" + host + '\'' +
", status=" + status +
", logPath='" + logPath + '\'' +
", executePath='" + executePath + '\'' +
'}';
}
} |
||||||
@ -1 +1 @@ |
|||||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task response command
*/
public class ExecuteTaskResponseCommand implements Serializable {
public ExecuteTaskResponseCommand() {
}
public ExecuteTaskResponseCommand(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
/**
* task instance id
*/
private int taskInstanceId;
/**
* status
*/
private int status;
/**
* end time
*/
private Date endTime;
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
return taskInstanceId;
}
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
}
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
kd">public int getTaskInstanceId() {
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
public int getStatus() {
kd">public int getTaskInstanceId() {
return status;
kd">public int getTaskInstanceId() {
public void setStatus(int status) {
return taskInstanceId;
return taskInstanceId;
kd">public int getTaskInstanceId() {
byte[] body = FastJsonSerializer.serialize(this);
return taskInstanceId;
}
return taskInstanceId;
}
return taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
return taskInstanceId;
public int getStatus() {
return taskInstanceId;
return status;
return taskInstanceId;
public void setStatus(int status) {
}
+
'}';
}
} |
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
/**
* execute task response command
*/
public class ExecuteTaskResponseCommand implements Serializable {
public ExecuteTaskResponseCommand() {
}
public ExecuteTaskResponseCommand(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
/**
* task instance id
*/
private int taskInstanceId;
/**
* status
*/
private int status;
/**
* end time
*/
private Date endTime;
}
kd">public int getTaskInstanceId() {
}
return taskInstanceId;
}
}
}
}
public void setTaskInstanceId(int taskInstanceId) {
}
this.taskInstanceId = taskInstanceId;
}
public int getStatus() {
*/
}
return status;
}
public void setTaskInstanceId(int taskInstanceId) {
}
public void setStatus(int status) {
}
}
public void setTaskInstanceId(int taskInstanceId) {
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
public int getStatus() {
return status;
kd">public int getTaskInstanceId() {
public void setStatus(int status) {
this.status = status;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
return status;
return endTime;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public void setEndTime(Date endTime) {
public void setTaskInstanceId(int taskInstanceId) {
kd">public int getTaskInstanceId() {
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public int getProcessId() {
return processId;
kd">public int getTaskInstanceId() {
}
public void setTaskInstanceId(int taskInstanceId) {
public void setProcessId(int processId) {
this.processId = processId;
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
public int getStatus() {
kd">public int getTaskInstanceId() {
public void setTaskInstanceId(int taskInstanceId) {
return status;
this.appIds = appIds;
kd">public int getTaskInstanceId() {
/**
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
kd">public int getTaskInstanceId() {
}
}
this.taskInstanceId = taskInstanceId;
return taskInstanceId;
this.taskInstanceId = taskInstanceId;
}
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
this.taskInstanceId = taskInstanceId;
public int getStatus() {
}
this.taskInstanceId = taskInstanceId;
return status;
public String toString() {
return "ExecuteTaskResponseCommand{" +
"taskInstanceId=" + taskInstanceId +
public int getStatus() {
return taskInstanceId;
public int getStatus() {
}
", processId=" + processId +
", appIds='" + appIds + '\'' +
'}';
}
} |
||||||
@ -1 +1 @@ |
|||||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
private int taskInstanceId;
private int processId;
private String host;
private String tenantCode;
private String logPath;
private String executePath;
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
}
public void setTenantCode(String tenantCode) {
this.tenantCode = tenantCode;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_REQUEST);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
} |
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* taskInstanceId
*/
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
kd">private int taskInstanceId;
private int processId;
kd">private int taskInstanceId;
private String host;
*/
kd">private int taskInstanceId;
private String tenantCode;
kd">private int taskInstanceId;
/**
kd">private int taskInstanceId;
private String logPath;
*/
kd">private int taskInstanceId;
/**
* tenantCode
*/
private String tenantCode;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
private String executePath;
public String getLogPath() {
return logPath;
}
public void setLogPath(String logPath) {
this.logPath = logPath;
}
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
}
public void setTenantCode(String tenantCode) {
this.tenantCode = tenantCode;
}
public String getExecutePath() {
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_REQUEST);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
} |
||||||
@ -0,0 +1 @@ |
|||||||
|
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
/**
* kill task response command
*/
public class KillTaskResponseCommand implements Serializable {
/**
* taskInstanceId
*/
private int taskInstanceId;
/**
* host
*/
private String host;
/**
* status
*/
private int status;
/**
* processId
*/
private int processId;
/**
* other resource manager appId , for example : YARN etc
*/
protected List<String> appIds;
public int getTaskInstanceId() {
return taskInstanceId;
}
public void setTaskInstanceId(int taskInstanceId) {
this.taskInstanceId = taskInstanceId;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public int getProcessId() {
return processId;
}
public void setProcessId(int processId) {
this.processId = processId;
}
public List<String> getAppIds() {
return appIds;
}
public void setAppIds(List<String> appIds) {
this.appIds = appIds;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.KILL_TASK_RESPONSE);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
@Override
public String toString() {
return "KillTaskResponseCommand{" +
"taskInstanceId=" + taskInstanceId +
", host='" + host + '\'' +
", status=" + status +
", processId=" + processId +
", appIds=" + appIds +
'}';
}
} |
||||||
@ -0,0 +1,48 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.cache; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.remote.entity.TaskExecutionContext; |
||||||
|
|
||||||
|
/** |
||||||
|
* TaskExecutionContextCacheManager |
||||||
|
*/ |
||||||
|
public interface TaskExecutionContextCacheManager { |
||||||
|
|
||||||
|
/** |
||||||
|
* get taskInstance by taskInstance id |
||||||
|
* |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @return taskInstance |
||||||
|
*/ |
||||||
|
TaskExecutionContext getByTaskInstanceId(Integer taskInstanceId); |
||||||
|
|
||||||
|
/** |
||||||
|
* cache taskInstance |
||||||
|
* |
||||||
|
* @param taskExecutionContext taskExecutionContext |
||||||
|
*/ |
||||||
|
void cacheTaskExecutionContext(TaskExecutionContext taskExecutionContext); |
||||||
|
|
||||||
|
/** |
||||||
|
* remove taskInstance by taskInstanceId |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
*/ |
||||||
|
void removeByTaskInstanceId(Integer taskInstanceId); |
||||||
|
} |
||||||
@ -0,0 +1,66 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.cache.impl; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.remote.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
|
||||||
|
/** |
||||||
|
* TaskExecutionContextCache |
||||||
|
*/ |
||||||
|
public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContextCacheManager { |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* taskInstance caceh |
||||||
|
*/ |
||||||
|
private Map<Integer,TaskExecutionContext> taskExecutionContextCache = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
/** |
||||||
|
* get taskInstance by taskInstance id |
||||||
|
* |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @return taskInstance |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public TaskExecutionContext getByTaskInstanceId(Integer taskInstanceId) { |
||||||
|
return taskExecutionContextCache.get(taskInstanceId); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* cache taskInstance |
||||||
|
* |
||||||
|
* @param taskExecutionContext taskExecutionContext |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void cacheTaskExecutionContext(TaskExecutionContext taskExecutionContext) { |
||||||
|
taskExecutionContextCache.put(taskExecutionContext.getTaskInstanceId(),taskExecutionContext); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* remove taskInstance by taskInstanceId |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void removeByTaskInstanceId(Integer taskInstanceId) { |
||||||
|
taskExecutionContextCache.remove(taskInstanceId); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,116 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.processor; |
||||||
|
|
||||||
|
|
||||||
|
import io.netty.channel.Channel; |
||||||
|
import io.netty.channel.ChannelFuture; |
||||||
|
import io.netty.channel.ChannelFutureListener; |
||||||
|
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
||||||
|
import org.apache.dolphinscheduler.remote.command.ExecuteTaskAckCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.command.ExecuteTaskResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.command.KillTaskResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
|
||||||
|
/** |
||||||
|
* taks callback service |
||||||
|
*/ |
||||||
|
public class KillTaskCallbackService { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(KillTaskCallbackService.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* remote channels |
||||||
|
*/ |
||||||
|
private static final ConcurrentHashMap<Integer, NettyRemoteChannel> REMOTE_CHANNELS = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
/** |
||||||
|
* netty remoting client |
||||||
|
*/ |
||||||
|
private final NettyRemotingClient nettyRemotingClient; |
||||||
|
|
||||||
|
|
||||||
|
public KillTaskCallbackService(){ |
||||||
|
final NettyClientConfig clientConfig = new NettyClientConfig(); |
||||||
|
this.nettyRemotingClient = new NettyRemotingClient(clientConfig); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* add callback channel |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @param channel channel |
||||||
|
*/ |
||||||
|
public void addRemoteChannel(int taskInstanceId, NettyRemoteChannel channel){ |
||||||
|
REMOTE_CHANNELS.put(taskInstanceId, channel); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get callback channel |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @return callback channel |
||||||
|
*/ |
||||||
|
public NettyRemoteChannel getRemoteChannel(int taskInstanceId){ |
||||||
|
NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(taskInstanceId); |
||||||
|
if(nettyRemoteChannel.isActive()){ |
||||||
|
return nettyRemoteChannel; |
||||||
|
} |
||||||
|
Channel newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost()); |
||||||
|
if(newChannel != null){ |
||||||
|
NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel, nettyRemoteChannel.getOpaque()); |
||||||
|
addRemoteChannel(taskInstanceId, remoteChannel); |
||||||
|
return remoteChannel; |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* remove callback channels |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
*/ |
||||||
|
public void remove(int taskInstanceId){ |
||||||
|
REMOTE_CHANNELS.remove(taskInstanceId); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* send result |
||||||
|
* |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @param killTaskResponseCommand killTaskResponseCommand |
||||||
|
*/ |
||||||
|
public void sendKillResult(int taskInstanceId, KillTaskResponseCommand killTaskResponseCommand){ |
||||||
|
NettyRemoteChannel nettyRemoteChannel = getRemoteChannel(taskInstanceId); |
||||||
|
if(nettyRemoteChannel == null){ |
||||||
|
//TODO
|
||||||
|
} else{ |
||||||
|
nettyRemoteChannel.writeAndFlush(killTaskResponseCommand.convert2Command()).addListener(new ChannelFutureListener(){ |
||||||
|
|
||||||
|
@Override |
||||||
|
public void operationComplete(ChannelFuture future) throws Exception { |
||||||
|
if(future.isSuccess()){ |
||||||
|
remove(taskInstanceId); |
||||||
|
return; |
||||||
|
} |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -1,365 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.server.worker.runner; |
|
||||||
|
|
||||||
import org.apache.curator.framework.recipes.locks.InterProcessMutex; |
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
|
||||||
import org.apache.dolphinscheduler.common.thread.Stopper; |
|
||||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.*; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.Tenant; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; |
|
||||||
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; |
|
||||||
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; |
|
||||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
|
||||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
|
||||||
import org.apache.dolphinscheduler.service.queue.ITaskQueue; |
|
||||||
import org.apache.dolphinscheduler.service.zk.AbstractZKClient; |
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import java.util.Arrays; |
|
||||||
import java.util.Date; |
|
||||||
import java.util.List; |
|
||||||
import java.util.concurrent.ExecutorService; |
|
||||||
import java.util.concurrent.ThreadPoolExecutor; |
|
||||||
|
|
||||||
/** |
|
||||||
* fetch task thread |
|
||||||
*/ |
|
||||||
public class FetchTaskThread implements Runnable{ |
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(FetchTaskThread.class); |
|
||||||
/** |
|
||||||
* set worker concurrent tasks |
|
||||||
*/ |
|
||||||
private final int taskNum; |
|
||||||
|
|
||||||
/** |
|
||||||
* zkWorkerClient |
|
||||||
*/ |
|
||||||
private final ZKWorkerClient zkWorkerClient; |
|
||||||
|
|
||||||
/** |
|
||||||
* task queue impl |
|
||||||
*/ |
|
||||||
protected ITaskQueue taskQueue; |
|
||||||
|
|
||||||
/** |
|
||||||
* process database access |
|
||||||
*/ |
|
||||||
private final ProcessService processService; |
|
||||||
|
|
||||||
/** |
|
||||||
* worker thread pool executor |
|
||||||
*/ |
|
||||||
private final ExecutorService workerExecService; |
|
||||||
|
|
||||||
/** |
|
||||||
* worker exec nums |
|
||||||
*/ |
|
||||||
private int workerExecNums; |
|
||||||
|
|
||||||
/** |
|
||||||
* task instance |
|
||||||
*/ |
|
||||||
private TaskInstance taskInstance; |
|
||||||
|
|
||||||
/** |
|
||||||
* task instance id |
|
||||||
*/ |
|
||||||
Integer taskInstId; |
|
||||||
|
|
||||||
/** |
|
||||||
* worker config |
|
||||||
*/ |
|
||||||
private WorkerConfig workerConfig; |
|
||||||
|
|
||||||
public FetchTaskThread(ZKWorkerClient zkWorkerClient, |
|
||||||
ProcessService processService, |
|
||||||
ITaskQueue taskQueue){ |
|
||||||
this.zkWorkerClient = zkWorkerClient; |
|
||||||
this.processService = processService; |
|
||||||
this.taskQueue = taskQueue; |
|
||||||
this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class); |
|
||||||
this.taskNum = workerConfig.getWorkerFetchTaskNum(); |
|
||||||
this.workerExecNums = workerConfig.getWorkerExecThreads(); |
|
||||||
// worker thread pool executor
|
|
||||||
this.workerExecService = ThreadUtils.newDaemonFixedThreadExecutor("Worker-Fetch-Task-Thread", workerExecNums); |
|
||||||
this.taskInstance = null; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Check if the task runs on this worker |
|
||||||
* @param taskInstance |
|
||||||
* @param host |
|
||||||
* @return |
|
||||||
*/ |
|
||||||
private boolean checkWorkerGroup(TaskInstance taskInstance, String host){ |
|
||||||
|
|
||||||
int taskWorkerGroupId = processService.getTaskWorkerGroupId(taskInstance); |
|
||||||
|
|
||||||
if(taskWorkerGroupId <= 0){ |
|
||||||
return true; |
|
||||||
} |
|
||||||
WorkerGroup workerGroup = processService.queryWorkerGroupById(taskWorkerGroupId); |
|
||||||
if(workerGroup == null ){ |
|
||||||
logger.info("task {} cannot find the worker group, use all worker instead.", taskInstance.getId()); |
|
||||||
return true; |
|
||||||
} |
|
||||||
String ips = workerGroup.getIpList(); |
|
||||||
if(StringUtils.isBlank(ips)){ |
|
||||||
logger.error("task:{} worker group:{} parameters(ip_list) is null, this task would be running on all workers", |
|
||||||
taskInstance.getId(), workerGroup.getId()); |
|
||||||
} |
|
||||||
String[] ipArray = ips.split(Constants.COMMA); |
|
||||||
List<String> ipList = Arrays.asList(ipArray); |
|
||||||
return ipList.contains(host); |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Override |
|
||||||
public void run() { |
|
||||||
logger.info("worker start fetch tasks..."); |
|
||||||
while (Stopper.isRunning()){ |
|
||||||
InterProcessMutex mutex = null; |
|
||||||
String currentTaskQueueStr = null; |
|
||||||
try { |
|
||||||
ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) workerExecService; |
|
||||||
//check memory and cpu usage and threads
|
|
||||||
boolean runCheckFlag = OSUtils.checkResource(workerConfig.getWorkerMaxCpuloadAvg(), workerConfig.getWorkerReservedMemory()) && checkThreadCount(poolExecutor); |
|
||||||
|
|
||||||
if(!runCheckFlag) { |
|
||||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
||||||
continue; |
|
||||||
} |
|
||||||
|
|
||||||
//whether have tasks, if no tasks , no need lock //get all tasks
|
|
||||||
boolean hasTask = taskQueue.hasTask(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); |
|
||||||
|
|
||||||
if (!hasTask){ |
|
||||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
||||||
continue; |
|
||||||
} |
|
||||||
// creating distributed locks, lock path /dolphinscheduler/lock/worker
|
|
||||||
mutex = zkWorkerClient.acquireZkLock(zkWorkerClient.getZkClient(), |
|
||||||
zkWorkerClient.getWorkerLockPath()); |
|
||||||
|
|
||||||
|
|
||||||
// task instance id str
|
|
||||||
List<String> taskQueueStrArr = taskQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskNum); |
|
||||||
|
|
||||||
for(String taskQueueStr : taskQueueStrArr){ |
|
||||||
|
|
||||||
currentTaskQueueStr = taskQueueStr; |
|
||||||
|
|
||||||
if (StringUtils.isEmpty(taskQueueStr)) { |
|
||||||
continue; |
|
||||||
} |
|
||||||
|
|
||||||
if (!checkThreadCount(poolExecutor)) { |
|
||||||
break; |
|
||||||
} |
|
||||||
|
|
||||||
// get task instance id
|
|
||||||
taskInstId = getTaskInstanceId(taskQueueStr); |
|
||||||
|
|
||||||
// mainly to wait for the master insert task to succeed
|
|
||||||
waitForTaskInstance(); |
|
||||||
|
|
||||||
taskInstance = processService.getTaskInstanceDetailByTaskId(taskInstId); |
|
||||||
|
|
||||||
// verify task instance is null
|
|
||||||
if (verifyTaskInstanceIsNull(taskInstance)) { |
|
||||||
logger.warn("remove task queue : {} due to taskInstance is null", taskQueueStr); |
|
||||||
processErrorTask(taskQueueStr); |
|
||||||
continue; |
|
||||||
} |
|
||||||
|
|
||||||
if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ |
|
||||||
continue; |
|
||||||
} |
|
||||||
|
|
||||||
// if process definition is null ,process definition already deleted
|
|
||||||
int userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId(); |
|
||||||
|
|
||||||
Tenant tenant = processService.getTenantForProcess( |
|
||||||
taskInstance.getProcessInstance().getTenantId(), |
|
||||||
userId); |
|
||||||
|
|
||||||
// verify tenant is null
|
|
||||||
if (verifyTenantIsNull(tenant)) { |
|
||||||
logger.warn("remove task queue : {} due to tenant is null", taskQueueStr); |
|
||||||
processErrorTask(taskQueueStr); |
|
||||||
continue; |
|
||||||
} |
|
||||||
|
|
||||||
// set queue for process instance, user-specified queue takes precedence over tenant queue
|
|
||||||
String userQueue = processService.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId()); |
|
||||||
taskInstance.getProcessInstance().setQueue(StringUtils.isEmpty(userQueue) ? tenant.getQueue() : userQueue); |
|
||||||
taskInstance.getProcessInstance().setTenantCode(tenant.getTenantCode()); |
|
||||||
|
|
||||||
logger.info("worker fetch taskId : {} from queue ", taskInstId); |
|
||||||
|
|
||||||
// local execute path
|
|
||||||
String execLocalPath = getExecLocalPath(); |
|
||||||
|
|
||||||
logger.info("task instance local execute path : {} ", execLocalPath); |
|
||||||
|
|
||||||
// init task
|
|
||||||
taskInstance.init(OSUtils.getHost(), |
|
||||||
new Date(), |
|
||||||
execLocalPath); |
|
||||||
|
|
||||||
// check and create users
|
|
||||||
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, |
|
||||||
tenant.getTenantCode()); |
|
||||||
|
|
||||||
logger.info("task : {} ready to submit to task scheduler thread",taskInstId); |
|
||||||
// submit task
|
|
||||||
// workerExecService.submit(new TaskExecuteThread(taskInstance, processService));
|
|
||||||
|
|
||||||
// remove node from zk
|
|
||||||
removeNodeFromTaskQueue(taskQueueStr); |
|
||||||
} |
|
||||||
|
|
||||||
}catch (Exception e){ |
|
||||||
processErrorTask(currentTaskQueueStr); |
|
||||||
logger.error("fetch task thread failure" ,e); |
|
||||||
}finally { |
|
||||||
AbstractZKClient.releaseMutex(mutex); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* process error task |
|
||||||
* |
|
||||||
* @param taskQueueStr task queue str |
|
||||||
*/ |
|
||||||
private void processErrorTask(String taskQueueStr){ |
|
||||||
// remove from zk
|
|
||||||
removeNodeFromTaskQueue(taskQueueStr); |
|
||||||
|
|
||||||
if (taskInstance != null){ |
|
||||||
processService.changeTaskState(ExecutionStatus.FAILURE, |
|
||||||
taskInstance.getStartTime(), |
|
||||||
taskInstance.getHost(), |
|
||||||
null, |
|
||||||
null, |
|
||||||
taskInstId); |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* remove node from task queue |
|
||||||
* |
|
||||||
* @param taskQueueStr task queue |
|
||||||
*/ |
|
||||||
private void removeNodeFromTaskQueue(String taskQueueStr){ |
|
||||||
taskQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, taskQueueStr); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* verify task instance is null |
|
||||||
* @param taskInstance |
|
||||||
* @return true if task instance is null |
|
||||||
*/ |
|
||||||
private boolean verifyTaskInstanceIsNull(TaskInstance taskInstance) { |
|
||||||
if (taskInstance == null ) { |
|
||||||
logger.error("task instance is null. task id : {} ", taskInstId); |
|
||||||
return true; |
|
||||||
} |
|
||||||
return false; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* verify tenant is null |
|
||||||
* |
|
||||||
* @param tenant tenant |
|
||||||
* @return true if tenant is null |
|
||||||
*/ |
|
||||||
private boolean verifyTenantIsNull(Tenant tenant) { |
|
||||||
if(tenant == null){ |
|
||||||
logger.error("tenant not exists,process instance id : {},task instance id : {}", |
|
||||||
taskInstance.getProcessInstance().getId(), |
|
||||||
taskInstance.getId()); |
|
||||||
return true; |
|
||||||
} |
|
||||||
return false; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get execute local path |
|
||||||
* |
|
||||||
* @return execute local path |
|
||||||
*/ |
|
||||||
private String getExecLocalPath(){ |
|
||||||
return FileUtils.getProcessExecDir(taskInstance.getProcessDefine().getProjectId(), |
|
||||||
taskInstance.getProcessDefine().getId(), |
|
||||||
taskInstance.getProcessInstance().getId(), |
|
||||||
taskInstance.getId()); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* check thread count |
|
||||||
* |
|
||||||
* @param poolExecutor pool executor |
|
||||||
* @return true if active count < worker exec nums |
|
||||||
*/ |
|
||||||
private boolean checkThreadCount(ThreadPoolExecutor poolExecutor) { |
|
||||||
int activeCount = poolExecutor.getActiveCount(); |
|
||||||
if (activeCount >= workerExecNums) { |
|
||||||
logger.info("thread insufficient , activeCount : {} , " + |
|
||||||
"workerExecNums : {}, will sleep : {} millis for thread resource", |
|
||||||
activeCount, |
|
||||||
workerExecNums, |
|
||||||
Constants.SLEEP_TIME_MILLIS); |
|
||||||
return false; |
|
||||||
} |
|
||||||
return true; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* wait for task instance exists, because of db action would be delayed. |
|
||||||
* |
|
||||||
* @throws Exception exception |
|
||||||
*/ |
|
||||||
private void waitForTaskInstance()throws Exception{ |
|
||||||
int retryTimes = 30; |
|
||||||
while (taskInstance == null && retryTimes > 0) { |
|
||||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
||||||
taskInstance = processService.findTaskInstanceById(taskInstId); |
|
||||||
retryTimes--; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get task instance id |
|
||||||
* |
|
||||||
* @param taskQueueStr task queue |
|
||||||
* @return task instance id |
|
||||||
*/ |
|
||||||
private int getTaskInstanceId(String taskQueueStr){ |
|
||||||
return Integer.parseInt(taskQueueStr.split(Constants.UNDERLINE)[3]); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,237 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.server.worker.task.dependent; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
|
||||||
import org.apache.dolphinscheduler.common.enums.DependentRelation; |
|
||||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
|
||||||
import org.apache.dolphinscheduler.common.model.DateInterval; |
|
||||||
import org.apache.dolphinscheduler.common.model.DependentItem; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DependentUtils; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
|
||||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
|
||||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import java.util.*; |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent item execute |
|
||||||
*/ |
|
||||||
public class DependentExecute { |
|
||||||
/** |
|
||||||
* process service |
|
||||||
*/ |
|
||||||
private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class); |
|
||||||
|
|
||||||
/** |
|
||||||
* depend item list |
|
||||||
*/ |
|
||||||
private List<DependentItem> dependItemList; |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent relation |
|
||||||
*/ |
|
||||||
private DependentRelation relation; |
|
||||||
|
|
||||||
/** |
|
||||||
* depend result |
|
||||||
*/ |
|
||||||
private DependResult modelDependResult = DependResult.WAITING; |
|
||||||
|
|
||||||
/** |
|
||||||
* depend result map |
|
||||||
*/ |
|
||||||
private Map<String, DependResult> dependResultMap = new HashMap<>(); |
|
||||||
|
|
||||||
/** |
|
||||||
* logger |
|
||||||
*/ |
|
||||||
private Logger logger = LoggerFactory.getLogger(DependentExecute.class); |
|
||||||
|
|
||||||
/** |
|
||||||
* constructor |
|
||||||
* @param itemList item list |
|
||||||
* @param relation relation |
|
||||||
*/ |
|
||||||
public DependentExecute(List<DependentItem> itemList, DependentRelation relation){ |
|
||||||
this.dependItemList = itemList; |
|
||||||
this.relation = relation; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get dependent item for one dependent item |
|
||||||
* @param dependentItem dependent item |
|
||||||
* @param currentTime current time |
|
||||||
* @return DependResult |
|
||||||
*/ |
|
||||||
public DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){ |
|
||||||
List<DateInterval> dateIntervals = DependentUtils.getDateIntervalList(currentTime, dependentItem.getDateValue()); |
|
||||||
return calculateResultForTasks(dependentItem, dateIntervals ); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* calculate dependent result for one dependent item. |
|
||||||
* @param dependentItem dependent item |
|
||||||
* @param dateIntervals date intervals |
|
||||||
* @return dateIntervals |
|
||||||
*/ |
|
||||||
private DependResult calculateResultForTasks(DependentItem dependentItem, |
|
||||||
List<DateInterval> dateIntervals) { |
|
||||||
DependResult result = DependResult.FAILED; |
|
||||||
for(DateInterval dateInterval : dateIntervals){ |
|
||||||
ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(), |
|
||||||
dateInterval); |
|
||||||
if(processInstance == null){ |
|
||||||
logger.error("cannot find the right process instance: definition id:{}, start:{}, end:{}", |
|
||||||
dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() ); |
|
||||||
return DependResult.FAILED; |
|
||||||
} |
|
||||||
if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){ |
|
||||||
result = getDependResultByState(processInstance.getState()); |
|
||||||
}else{ |
|
||||||
TaskInstance taskInstance = null; |
|
||||||
List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); |
|
||||||
|
|
||||||
for(TaskInstance task : taskInstanceList){ |
|
||||||
if(task.getName().equals(dependentItem.getDepTasks())){ |
|
||||||
taskInstance = task; |
|
||||||
break; |
|
||||||
} |
|
||||||
} |
|
||||||
if(taskInstance == null){ |
|
||||||
// cannot find task in the process instance
|
|
||||||
// maybe because process instance is running or failed.
|
|
||||||
result = getDependResultByState(processInstance.getState()); |
|
||||||
}else{ |
|
||||||
result = getDependResultByState(taskInstance.getState()); |
|
||||||
} |
|
||||||
} |
|
||||||
if(result != DependResult.SUCCESS){ |
|
||||||
break; |
|
||||||
} |
|
||||||
} |
|
||||||
return result; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* find the last one process instance that : |
|
||||||
* 1. manual run and finish between the interval |
|
||||||
* 2. schedule run and schedule time between the interval |
|
||||||
* @param definitionId definition id |
|
||||||
* @param dateInterval date interval |
|
||||||
* @return ProcessInstance |
|
||||||
*/ |
|
||||||
private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) { |
|
||||||
|
|
||||||
ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval); |
|
||||||
if(runningProcess != null){ |
|
||||||
return runningProcess; |
|
||||||
} |
|
||||||
|
|
||||||
ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval( |
|
||||||
definitionId, dateInterval |
|
||||||
); |
|
||||||
|
|
||||||
ProcessInstance lastManualProcess = processService.findLastManualProcessInterval( |
|
||||||
definitionId, dateInterval |
|
||||||
); |
|
||||||
|
|
||||||
if(lastManualProcess ==null){ |
|
||||||
return lastSchedulerProcess; |
|
||||||
} |
|
||||||
if(lastSchedulerProcess == null){ |
|
||||||
return lastManualProcess; |
|
||||||
} |
|
||||||
|
|
||||||
return (lastManualProcess.getEndTime().after(lastSchedulerProcess.getEndTime()))? |
|
||||||
lastManualProcess : lastSchedulerProcess; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get dependent result by task/process instance state |
|
||||||
* @param state state |
|
||||||
* @return DependResult |
|
||||||
*/ |
|
||||||
private DependResult getDependResultByState(ExecutionStatus state) { |
|
||||||
|
|
||||||
if(state.typeIsRunning() || state == ExecutionStatus.SUBMITTED_SUCCESS || state == ExecutionStatus.WAITTING_THREAD){ |
|
||||||
return DependResult.WAITING; |
|
||||||
}else if(state.typeIsSuccess()){ |
|
||||||
return DependResult.SUCCESS; |
|
||||||
}else{ |
|
||||||
return DependResult.FAILED; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* judge depend item finished |
|
||||||
* @param currentTime current time |
|
||||||
* @return boolean |
|
||||||
*/ |
|
||||||
public boolean finish(Date currentTime){ |
|
||||||
if(modelDependResult == DependResult.WAITING){ |
|
||||||
modelDependResult = getModelDependResult(currentTime); |
|
||||||
return false; |
|
||||||
} |
|
||||||
return true; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get model depend result |
|
||||||
* @param currentTime current time |
|
||||||
* @return DependResult |
|
||||||
*/ |
|
||||||
public DependResult getModelDependResult(Date currentTime){ |
|
||||||
|
|
||||||
List<DependResult> dependResultList = new ArrayList<>(); |
|
||||||
|
|
||||||
for(DependentItem dependentItem : dependItemList){ |
|
||||||
DependResult dependResult = getDependResultForItem(dependentItem, currentTime); |
|
||||||
if(dependResult != DependResult.WAITING){ |
|
||||||
dependResultMap.put(dependentItem.getKey(), dependResult); |
|
||||||
} |
|
||||||
dependResultList.add(dependResult); |
|
||||||
} |
|
||||||
modelDependResult = DependentUtils.getDependResultForRelation( |
|
||||||
this.relation, dependResultList |
|
||||||
); |
|
||||||
return modelDependResult; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get dependent item result |
|
||||||
* @param item item |
|
||||||
* @param currentTime current time |
|
||||||
* @return DependResult |
|
||||||
*/ |
|
||||||
public DependResult getDependResultForItem(DependentItem item, Date currentTime){ |
|
||||||
String key = item.getKey(); |
|
||||||
if(dependResultMap.containsKey(key)){ |
|
||||||
return dependResultMap.get(key); |
|
||||||
} |
|
||||||
return getDependentResultForItem(item, currentTime); |
|
||||||
} |
|
||||||
|
|
||||||
public Map<String, DependResult> getDependResultMap(){ |
|
||||||
return dependResultMap; |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -1,191 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.server.worker.task.dependent; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
|
||||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
|
||||||
import org.apache.dolphinscheduler.common.model.DependentTaskModel; |
|
||||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
|
||||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; |
|
||||||
import org.apache.dolphinscheduler.common.thread.Stopper; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DependentUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
|
||||||
import org.apache.dolphinscheduler.server.worker.task.AbstractTask; |
|
||||||
import org.apache.dolphinscheduler.server.worker.task.TaskProps; |
|
||||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
|
||||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
|
||||||
import org.slf4j.Logger; |
|
||||||
|
|
||||||
import java.util.*; |
|
||||||
|
|
||||||
import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; |
|
||||||
|
|
||||||
/** |
|
||||||
* Dependent Task |
|
||||||
*/ |
|
||||||
public class DependentTask extends AbstractTask { |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent task list |
|
||||||
*/ |
|
||||||
private List<DependentExecute> dependentTaskList = new ArrayList<>(); |
|
||||||
|
|
||||||
/** |
|
||||||
* depend item result map |
|
||||||
* save the result to log file |
|
||||||
*/ |
|
||||||
private Map<String, DependResult> dependResultMap = new HashMap<>(); |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent parameters |
|
||||||
*/ |
|
||||||
private DependentParameters dependentParameters; |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent date |
|
||||||
*/ |
|
||||||
private Date dependentDate; |
|
||||||
|
|
||||||
/** |
|
||||||
* process service |
|
||||||
*/ |
|
||||||
private ProcessService processService; |
|
||||||
|
|
||||||
/** |
|
||||||
* constructor |
|
||||||
* @param props props |
|
||||||
* @param logger logger |
|
||||||
*/ |
|
||||||
public DependentTask(TaskProps props, Logger logger) { |
|
||||||
super(props, logger); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public void init(){ |
|
||||||
logger.info("dependent task initialize"); |
|
||||||
|
|
||||||
this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(), |
|
||||||
DependentParameters.class); |
|
||||||
|
|
||||||
for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){ |
|
||||||
this.dependentTaskList.add(new DependentExecute( |
|
||||||
taskModel.getDependItemList(), taskModel.getRelation())); |
|
||||||
} |
|
||||||
|
|
||||||
this.processService = SpringApplicationContext.getBean(ProcessService.class); |
|
||||||
|
|
||||||
if(taskProps.getScheduleTime() != null){ |
|
||||||
this.dependentDate = taskProps.getScheduleTime(); |
|
||||||
}else{ |
|
||||||
this.dependentDate = taskProps.getTaskStartTime(); |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public void handle() throws Exception { |
|
||||||
// set the name of the current thread
|
|
||||||
String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); |
|
||||||
Thread.currentThread().setName(threadLoggerInfoName); |
|
||||||
|
|
||||||
try{ |
|
||||||
TaskInstance taskInstance = null; |
|
||||||
while(Stopper.isRunning()){ |
|
||||||
taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstanceId()); |
|
||||||
|
|
||||||
if(taskInstance == null){ |
|
||||||
exitStatusCode = -1; |
|
||||||
break; |
|
||||||
} |
|
||||||
|
|
||||||
if(taskInstance.getState() == ExecutionStatus.KILL){ |
|
||||||
this.cancel = true; |
|
||||||
} |
|
||||||
|
|
||||||
if(this.cancel || allDependentTaskFinish()){ |
|
||||||
break; |
|
||||||
} |
|
||||||
|
|
||||||
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
||||||
} |
|
||||||
|
|
||||||
if(cancel){ |
|
||||||
exitStatusCode = Constants.EXIT_CODE_KILL; |
|
||||||
}else{ |
|
||||||
DependResult result = getTaskDependResult(); |
|
||||||
exitStatusCode = (result == DependResult.SUCCESS) ? |
|
||||||
Constants.EXIT_CODE_SUCCESS : Constants.EXIT_CODE_FAILURE; |
|
||||||
} |
|
||||||
}catch (Exception e){ |
|
||||||
logger.error(e.getMessage(),e); |
|
||||||
exitStatusCode = -1; |
|
||||||
throw e; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get dependent result |
|
||||||
* @return DependResult |
|
||||||
*/ |
|
||||||
private DependResult getTaskDependResult(){ |
|
||||||
List<DependResult> dependResultList = new ArrayList<>(); |
|
||||||
for(DependentExecute dependentExecute : dependentTaskList){ |
|
||||||
DependResult dependResult = dependentExecute.getModelDependResult(dependentDate); |
|
||||||
dependResultList.add(dependResult); |
|
||||||
} |
|
||||||
DependResult result = DependentUtils.getDependResultForRelation( |
|
||||||
this.dependentParameters.getRelation(), dependResultList |
|
||||||
); |
|
||||||
return result; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* judge all dependent tasks finish |
|
||||||
* @return whether all dependent tasks finish |
|
||||||
*/ |
|
||||||
private boolean allDependentTaskFinish(){ |
|
||||||
boolean finish = true; |
|
||||||
for(DependentExecute dependentExecute : dependentTaskList){ |
|
||||||
for(Map.Entry<String, DependResult> entry: dependentExecute.getDependResultMap().entrySet()) { |
|
||||||
if(!dependResultMap.containsKey(entry.getKey())){ |
|
||||||
dependResultMap.put(entry.getKey(), entry.getValue()); |
|
||||||
//save depend result to log
|
|
||||||
logger.info("dependent item complete {} {},{}", |
|
||||||
DEPENDENT_SPLIT, entry.getKey(), entry.getValue().toString()); |
|
||||||
} |
|
||||||
} |
|
||||||
if(!dependentExecute.finish(dependentDate)){ |
|
||||||
finish = false; |
|
||||||
} |
|
||||||
} |
|
||||||
return finish; |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
@Override |
|
||||||
public void cancelApplication(boolean cancelApplication) throws Exception { |
|
||||||
// cancel process
|
|
||||||
this.cancel = true; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public AbstractParameters getParameters() { |
|
||||||
return null; |
|
||||||
} |
|
||||||
} |
|
||||||
Loading…
Reference in new issue