|
|
|
|
@ -21,170 +21,108 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelect
|
|
|
|
|
import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable; |
|
|
|
|
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; |
|
|
|
|
|
|
|
|
|
import org.springframework.boot.context.properties.ConfigurationProperties; |
|
|
|
|
import org.springframework.boot.context.properties.EnableConfigurationProperties; |
|
|
|
|
import org.springframework.stereotype.Component; |
|
|
|
|
import java.time.Duration; |
|
|
|
|
|
|
|
|
|
@Component |
|
|
|
|
@EnableConfigurationProperties |
|
|
|
|
@ConfigurationProperties("master") |
|
|
|
|
public class MasterConfig { |
|
|
|
|
import org.springframework.boot.context.properties.ConfigurationProperties; |
|
|
|
|
import org.springframework.context.annotation.Configuration; |
|
|
|
|
import org.springframework.validation.Errors; |
|
|
|
|
import org.springframework.validation.Validator; |
|
|
|
|
import org.springframework.validation.annotation.Validated; |
|
|
|
|
|
|
|
|
|
import lombok.Data; |
|
|
|
|
|
|
|
|
|
@Data |
|
|
|
|
@Validated |
|
|
|
|
@Configuration |
|
|
|
|
@ConfigurationProperties(prefix = "master") |
|
|
|
|
public class MasterConfig implements Validator { |
|
|
|
|
/** |
|
|
|
|
* The master RPC server listen port. |
|
|
|
|
*/ |
|
|
|
|
private int listenPort; |
|
|
|
|
private int listenPort = 5678; |
|
|
|
|
/** |
|
|
|
|
* The max batch size used to fetch command from database. |
|
|
|
|
*/ |
|
|
|
|
private int fetchCommandNum; |
|
|
|
|
private int fetchCommandNum = 10; |
|
|
|
|
/** |
|
|
|
|
* The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum. |
|
|
|
|
*/ |
|
|
|
|
private int preExecThreads; |
|
|
|
|
private int preExecThreads = 10; |
|
|
|
|
/** |
|
|
|
|
* todo: We may need to split the process/task into different thread size. |
|
|
|
|
* The thread number used to handle processInstance and task event. |
|
|
|
|
* Will create two thread poll to execute {@link WorkflowExecuteRunnable} and {@link TaskExecuteRunnable}. |
|
|
|
|
*/ |
|
|
|
|
private int execThreads; |
|
|
|
|
private int execThreads = 10; |
|
|
|
|
/** |
|
|
|
|
* The task dispatch thread pool size. |
|
|
|
|
*/ |
|
|
|
|
private int dispatchTaskNumber; |
|
|
|
|
private int dispatchTaskNumber = 3; |
|
|
|
|
/** |
|
|
|
|
* Worker select strategy. |
|
|
|
|
*/ |
|
|
|
|
private HostSelector hostSelector; |
|
|
|
|
private HostSelector hostSelector = HostSelector.LOWER_WEIGHT; |
|
|
|
|
/** |
|
|
|
|
* Master heart beat task execute interval. |
|
|
|
|
*/ |
|
|
|
|
private int heartbeatInterval; |
|
|
|
|
private Duration heartbeatInterval = Duration.ofSeconds(10); |
|
|
|
|
/** |
|
|
|
|
* task submit max retry times. |
|
|
|
|
*/ |
|
|
|
|
private int taskCommitRetryTimes; |
|
|
|
|
private int taskCommitRetryTimes = 5; |
|
|
|
|
/** |
|
|
|
|
* task submit retry interval/ms. |
|
|
|
|
* task submit retry interval. |
|
|
|
|
*/ |
|
|
|
|
private int taskCommitInterval; |
|
|
|
|
private Duration taskCommitInterval = Duration.ofSeconds(1); |
|
|
|
|
/** |
|
|
|
|
* state wheel check interval/ms, if this value is bigger, may increase the delay of task/processInstance. |
|
|
|
|
* state wheel check interval, if this value is bigger, may increase the delay of task/processInstance. |
|
|
|
|
*/ |
|
|
|
|
private int stateWheelInterval; |
|
|
|
|
private double maxCpuLoadAvg; |
|
|
|
|
private double reservedMemory; |
|
|
|
|
private int failoverInterval; |
|
|
|
|
private boolean killYarnJobWhenTaskFailover; |
|
|
|
|
|
|
|
|
|
public int getListenPort() { |
|
|
|
|
return listenPort; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setListenPort(int listenPort) { |
|
|
|
|
this.listenPort = listenPort; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getFetchCommandNum() { |
|
|
|
|
return fetchCommandNum; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setFetchCommandNum(int fetchCommandNum) { |
|
|
|
|
this.fetchCommandNum = fetchCommandNum; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getPreExecThreads() { |
|
|
|
|
return preExecThreads; |
|
|
|
|
} |
|
|
|
|
private Duration stateWheelInterval = Duration.ofMillis(5); |
|
|
|
|
private double maxCpuLoadAvg = -1; |
|
|
|
|
private double reservedMemory = 0.3; |
|
|
|
|
private Duration failoverInterval = Duration.ofMinutes(10); |
|
|
|
|
private boolean killYarnJobWhenTaskFailover = true; |
|
|
|
|
|
|
|
|
|
public void setPreExecThreads(int preExecThreads) { |
|
|
|
|
this.preExecThreads = preExecThreads; |
|
|
|
|
@Override |
|
|
|
|
public boolean supports(Class<?> clazz) { |
|
|
|
|
return MasterConfig.class.isAssignableFrom(clazz); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getExecThreads() { |
|
|
|
|
return execThreads; |
|
|
|
|
@Override |
|
|
|
|
public void validate(Object target, Errors errors) { |
|
|
|
|
MasterConfig masterConfig = (MasterConfig) target; |
|
|
|
|
if (masterConfig.getListenPort() <= 0) { |
|
|
|
|
errors.rejectValue("listen-port", null, "is invalidated"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setExecThreads(int execThreads) { |
|
|
|
|
this.execThreads = execThreads; |
|
|
|
|
if (masterConfig.getFetchCommandNum() <= 0) { |
|
|
|
|
errors.rejectValue("fetch-command-num", null, "should be a positive value"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getDispatchTaskNumber() { |
|
|
|
|
return dispatchTaskNumber; |
|
|
|
|
if (masterConfig.getPreExecThreads() <= 0) { |
|
|
|
|
errors.rejectValue("per-exec-threads", null, "should be a positive value"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setDispatchTaskNumber(int dispatchTaskNumber) { |
|
|
|
|
this.dispatchTaskNumber = dispatchTaskNumber; |
|
|
|
|
if (masterConfig.getExecThreads() <= 0) { |
|
|
|
|
errors.rejectValue("exec-threads", null, "should be a positive value"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public HostSelector getHostSelector() { |
|
|
|
|
return hostSelector; |
|
|
|
|
if (masterConfig.getDispatchTaskNumber() <= 0) { |
|
|
|
|
errors.rejectValue("dispatch-task-number", null, "should be a positive value"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setHostSelector(HostSelector hostSelector) { |
|
|
|
|
this.hostSelector = hostSelector; |
|
|
|
|
if (masterConfig.getHeartbeatInterval().toMillis() < 0) { |
|
|
|
|
errors.rejectValue("heartbeat-interval", null, "should be a valid duration"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getHeartbeatInterval() { |
|
|
|
|
return heartbeatInterval; |
|
|
|
|
if (masterConfig.getTaskCommitRetryTimes() <= 0) { |
|
|
|
|
errors.rejectValue("task-commit-retry-times", null, "should be a positive value"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setHeartbeatInterval(int heartbeatInterval) { |
|
|
|
|
this.heartbeatInterval = heartbeatInterval; |
|
|
|
|
if (masterConfig.getTaskCommitInterval().toMillis() <= 0) { |
|
|
|
|
errors.rejectValue("task-commit-interval", null, "should be a valid duration"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getTaskCommitRetryTimes() { |
|
|
|
|
return taskCommitRetryTimes; |
|
|
|
|
if (masterConfig.getStateWheelInterval().toMillis() <= 0) { |
|
|
|
|
errors.rejectValue("state-wheel-interval", null, "should be a valid duration"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setTaskCommitRetryTimes(int taskCommitRetryTimes) { |
|
|
|
|
this.taskCommitRetryTimes = taskCommitRetryTimes; |
|
|
|
|
if (masterConfig.getFailoverInterval().toMillis() <= 0) { |
|
|
|
|
errors.rejectValue("failover-interval", null, "should be a valid duration"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getTaskCommitInterval() { |
|
|
|
|
return taskCommitInterval; |
|
|
|
|
if (masterConfig.getMaxCpuLoadAvg() <= 0) { |
|
|
|
|
masterConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setTaskCommitInterval(int taskCommitInterval) { |
|
|
|
|
this.taskCommitInterval = taskCommitInterval; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getStateWheelInterval() { |
|
|
|
|
return stateWheelInterval; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setStateWheelInterval(int stateWheelInterval) { |
|
|
|
|
this.stateWheelInterval = stateWheelInterval; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public double getMaxCpuLoadAvg() { |
|
|
|
|
return maxCpuLoadAvg > 0 ? maxCpuLoadAvg : Runtime.getRuntime().availableProcessors() * 2; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setMaxCpuLoadAvg(double maxCpuLoadAvg) { |
|
|
|
|
this.maxCpuLoadAvg = maxCpuLoadAvg; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public double getReservedMemory() { |
|
|
|
|
return reservedMemory; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setReservedMemory(double reservedMemory) { |
|
|
|
|
this.reservedMemory = reservedMemory; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int getFailoverInterval() { |
|
|
|
|
return failoverInterval; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setFailoverInterval(int failoverInterval) { |
|
|
|
|
this.failoverInterval = failoverInterval; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public boolean isKillYarnJobWhenTaskFailover() { |
|
|
|
|
return killYarnJobWhenTaskFailover; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void setKillYarnJobWhenTaskFailover(boolean killYarnJobWhenTaskFailover) { |
|
|
|
|
this.killYarnJobWhenTaskFailover = killYarnJobWhenTaskFailover; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
Wenjun Ruan
2 years ago