DolphinScheduler/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.dolphinscheduler.server.master.config;

import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector;
import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;

import org.apache.commons.lang3.StringUtils;

import java.time.Duration;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.validation.Errors;
import org.springframework.validation.Validator;
import org.springframework.validation.annotation.Validated;

@Data
@Validated
@Configuration
@ConfigurationProperties(prefix = "master")
@Slf4j
public class MasterConfig implements Validator {

    /**
     * The master RPC server listen port.
     */
    private int listenPort = 5678;
    /**
     * The max batch size used to fetch command from database.
     */
    private int fetchCommandNum = 10;
    /**
     * The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum.
     */
    private int preExecThreads = 10;
    /**
     * todo: We may need to split the process/task into different thread size.
     * The thread number used to handle processInstance and task event.
     * Will create two thread poll to execute {@link WorkflowExecuteRunnable} and {@link TaskExecuteRunnable}.
     */
    private int execThreads = 10;

    // todo: change to sync thread pool/ async thread pool ?
    private int masterSyncTaskExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors();

    private int masterAsyncTaskExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors();
    /**
     * The task dispatch thread pool size.
     */
    private int dispatchTaskNumber = 3;
    /**
     * Worker select strategy.
     */
    private HostSelector hostSelector = HostSelector.LOWER_WEIGHT;
    /**
     * Master heart beat task execute interval.
     */
    private Duration heartbeatInterval = Duration.ofSeconds(10);
    /**
     * task submit max retry times.
     */
    private int taskCommitRetryTimes = 5;
    /**
     * task submit retry interval.
     */
    private Duration taskCommitInterval = Duration.ofSeconds(1);
    /**
     * state wheel check interval, if this value is bigger, may increase the delay of task/processInstance.
     */
    private Duration stateWheelInterval = Duration.ofMillis(5);
    private double maxCpuLoadAvg = 1;
    private double reservedMemory = 0.1;
    private Duration failoverInterval = Duration.ofMinutes(10);
    private boolean killApplicationWhenTaskFailover = true;
    private ConnectStrategyProperties registryDisconnectStrategy = new ConnectStrategyProperties();

    private Duration workerGroupRefreshInterval = Duration.ofSeconds(10L);

    // ip:listenPort
    private String masterAddress;

    // /nodes/master/ip:listenPort
    private String masterRegistryPath;

    @Override
    public boolean supports(Class<?> clazz) {
        return MasterConfig.class.isAssignableFrom(clazz);
    }

    @Override
    public void validate(Object target, Errors errors) {
        MasterConfig masterConfig = (MasterConfig) target;
        if (masterConfig.getListenPort() <= 0) {
            errors.rejectValue("listen-port", null, "is invalidated");
        }
        if (masterConfig.getFetchCommandNum() <= 0) {
            errors.rejectValue("fetch-command-num", null, "should be a positive value");
        }
        if (masterConfig.getPreExecThreads() <= 0) {
            errors.rejectValue("per-exec-threads", null, "should be a positive value");
        }
        if (masterConfig.getExecThreads() <= 0) {
            errors.rejectValue("exec-threads", null, "should be a positive value");
        }
        if (masterConfig.getDispatchTaskNumber() <= 0) {
            errors.rejectValue("dispatch-task-number", null, "should be a positive value");
        }
        if (masterConfig.getHeartbeatInterval().toMillis() < 0) {
            errors.rejectValue("heartbeat-interval", null, "should be a valid duration");
        }
        if (masterConfig.getTaskCommitRetryTimes() <= 0) {
            errors.rejectValue("task-commit-retry-times", null, "should be a positive value");
        }
        if (masterConfig.getTaskCommitInterval().toMillis() <= 0) {
            errors.rejectValue("task-commit-interval", null, "should be a valid duration");
        }
        if (masterConfig.getStateWheelInterval().toMillis() <= 0) {
            errors.rejectValue("state-wheel-interval", null, "should be a valid duration");
        }
        if (masterConfig.getFailoverInterval().toMillis() <= 0) {
            errors.rejectValue("failover-interval", null, "should be a valid duration");
        }
        if (masterConfig.getMaxCpuLoadAvg() <= 0) {
            masterConfig.setMaxCpuLoadAvg(100);
        }
        if (masterConfig.getReservedMemory() <= 0) {
            masterConfig.setReservedMemory(100);
        }

        if (masterConfig.getWorkerGroupRefreshInterval().getSeconds() < 10) {
            errors.rejectValue("worker-group-refresh-interval", null, "should >= 10s");
        }
        if (StringUtils.isEmpty(masterConfig.getMasterAddress())) {
            masterConfig.setMasterAddress(NetUtils.getAddr(masterConfig.getListenPort()));
        }

        masterConfig.setMasterRegistryPath(
                RegistryNodeType.MASTER.getRegistryPath() + "/" + masterConfig.getMasterAddress());
        printConfig();
    }

    private void printConfig() {
        String config =
                "\n****************************Master Configuration**************************************" +
                        "\n  listen-port -> " + listenPort +
                        "\n  fetch-command-num -> " + fetchCommandNum +
                        "\n  pre-exec-threads -> " + preExecThreads +
                        "\n  exec-threads -> " + execThreads +
                        "\n  dispatch-task-number -> " + dispatchTaskNumber +
                        "\n  host-selector -> " + hostSelector +
                        "\n  heartbeat-interval -> " + heartbeatInterval +
                        "\n  task-commit-retry-times -> " + taskCommitRetryTimes +
                        "\n  task-commit-interval -> " + taskCommitInterval +
                        "\n  state-wheel-interval -> " + stateWheelInterval +
                        "\n  max-cpu-load-avg -> " + maxCpuLoadAvg +
                        "\n  reserved-memory -> " + reservedMemory +
                        "\n  failover-interval -> " + failoverInterval +
                        "\n  kill-application-when-task-failover -> " + killApplicationWhenTaskFailover +
                        "\n  registry-disconnect-strategy -> " + registryDisconnectStrategy +
                        "\n  master-address -> " + masterAddress +
                        "\n  master-registry-path: " + masterRegistryPath +
                        "\n  worker-group-refresh-interval: " + workerGroupRefreshInterval +
                        "\n****************************Master Configuration**************************************";
        log.info(config);
    }
}