add multiply settings for application.yml (#973)

* update english documents

* refactor zk client

* update documents

* update zkclient

* update zkclient

* update documents

* add architecture-design

* change i18n

* update i18n

* update english documents

* add architecture-design

* update english documents

* update en-US documents

* add architecture-design

* update demo site

* add mybatis plus model

* modify mybatisplus

* modify mybatisplus

* change interface by mybatisplus

* add unit test

* refactor dao interface.

* add unit test for dao...

* add unit test for dao...

* add unit test for dao...

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update some dao bugs

* update for some bugs

* update some bugs

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update

* update

* add multiply settings for application.yml

* add multiply settings for application.yml

* revert

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java

@@ -74,7 +74,7 @@ public class TenantService extends BaseService{
       return result;
     }
 
-    if (!checkTenant(tenantCode)){
+    if (checkTenantExists(tenantCode)){
       putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode);
       return result;
     }
@@ -168,7 +168,7 @@ public class TenantService extends BaseService{
      * if the tenant code is modified, the original resource needs to be copied to the new tenant.
      */
     if (!tenant.getTenantCode().equals(tenantCode)){
-      if (checkTenant(tenantCode)){
+      if (checkTenantExists(tenantCode)){
         // if hdfs startup
         if (PropertyUtils.getResUploadStartupState()){
           String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources";
@@ -278,8 +278,7 @@ public class TenantService extends BaseService{
    */
   public Result verifyTenantCode(String tenantCode) {
     Result result=new Result();
-    Tenant tenant = tenantMapper.queryByTenantCode(tenantCode);
-    if (tenant != null) {
+    if (checkTenantExists(tenantCode)) {
       logger.error("tenant {} has exist, can't create again.", tenantCode);
       putMsg(result, Status.TENANT_NAME_EXIST);
     }else{
@@ -295,7 +294,8 @@ public class TenantService extends BaseService{
    * @param tenantCode
    * @return
    */
-  private boolean checkTenant(String tenantCode) {
-    return tenantMapper.queryByTenantCode(tenantCode) == null ? true : false;
+  private boolean checkTenantExists(String tenantCode) {
+      List<Tenant> tenants = tenantMapper.queryByTenantCode(tenantCode);
+      return (tenants != null && tenants.size() > 0);
   }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@@ -132,6 +132,9 @@ public class DateUtils {
      * @return
      */
     public static long differSec(Date d1, Date d2) {
+        if(d1 == null || d2 == null){
+            return 0;
+        }
         return (long) Math.ceil(differMs(d1, d2) / 1000.0);
     }
 

dolphinscheduler-common/src/main/resources/quartz.properties

@@ -32,8 +32,8 @@ org.quartz.jobStore.dataSource = myDs
 #============================================================================
 org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider
 org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
-org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
-org.quartz.dataSource.myDs.user = xx
-org.quartz.dataSource.myDs.password = xx
+org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.220.188:3306/dolphinscheduler_bl?characterEncoding=utf8
+org.quartz.dataSource.myDs.user = root
+org.quartz.dataSource.myDs.password = root@123
 org.quartz.dataSource.myDs.maxConnections = 10
 org.quartz.dataSource.myDs.validationQuery = select 1

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java

@@ -17,6 +17,7 @@
 package org.apache.dolphinscheduler.dao;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.config.YmlConfig;
 import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
@@ -41,25 +42,26 @@ public class MonitorDBDao {
     /**
      * 加载配置文件
      */
-    private static Configuration conf;
+//    private static Configuration conf;
+
+//    static {
+//        try {
+//            conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
+//        }catch (ConfigurationException e){
+//            logger.error("load configuration excetpion",e);
+//            System.exit(1);
+//        }
+//    }
 
-    static {
-        try {
-            conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
-        }catch (ConfigurationException e){
-            logger.error("load configuration excetpion",e);
-            System.exit(1);
-        }
-    }
 
     /**
      * create connection
      * @return
      */
     private static Connection getConn() {
-        String url = conf.getString(Constants.SPRING_DATASOURCE_URL);
-        String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
-        String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
+        String url =  YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_URL);
+        String username = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_USERNAME);
+        String password = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_PASSWORD);
         Connection conn = null;
         try {
             //classloader,load driver

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java

@@ -146,17 +146,17 @@ public class ProcessDao extends AbstractBaseDao {
                 delCommandByid(command.getId());
                 saveErrorCommand(command, "process instance is null");
                 return null;
-            }else if(!checkThreadNum(command, validThreadNum)){
-                    logger.info("there is not enough thread for this command: {}",command.toString() );
-                    return setWaitingThreadProcess(command, processInstance);
-            }else{
-                    processInstance.setCommandType(command.getCommandType());
-                    processInstance.addHistoryCmd(command.getCommandType());
-                    saveProcessInstance(processInstance);
-                    this.setSubProcessParam(processInstance);
-                    delCommandByid(command.getId());
-                    return processInstance;
             }
+            if(!checkThreadNum(command, validThreadNum)){
+                logger.info("there is not enough thread for this command: {}",command.toString() );
+                return setWaitingThreadProcess(command, processInstance);
+            }
+            processInstance.setCommandType(command.getCommandType());
+            processInstance.addHistoryCmd(command.getCommandType());
+            saveProcessInstance(processInstance);
+            this.setSubProcessParam(processInstance);
+            delCommandByid(command.getId());
+            return processInstance;
         }catch (Exception e){
             logger.error("scan command error ", e);
             saveErrorCommand(command, e.toString());

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java

@@ -27,7 +27,7 @@ public interface TenantMapper extends BaseMapper<Tenant> {
 
     Tenant queryById(@Param("tenantId") int tenantId);
 
-    Tenant queryByTenantCode(@Param("tenantCode") String tenantCode);
+    List<Tenant> queryByTenantCode(@Param("tenantCode") String tenantCode);
 
     IPage<Tenant> queryTenantPaging(IPage<Tenant> page,
                                     @Param("searchVal") String searchVal);

dolphinscheduler-dao/src/main/resources/application.yml

@@ -1,14 +1,38 @@
 # mysql
 spring:
-   datasource:
-    driver-class-name: org.postgresql.Driver
-#    driver-class-name:  com.mysql.jdbc.Driver
-    url: jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
-#    url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-    username: root
-    password: root@123
-#    platform:
-    continue-on-error: true
+
+    datasource:
+        url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+        username: root
+        password: root@123
+        #
+        type: com.alibaba.druid.pool.DruidDataSource
+        driver-class-name: com.mysql.jdbc.Driver
+        initialSize: 5
+        minIdle: 5
+        maxActive: 20
+        maxWait: 60000
+        timeBetweenEvictionRunsMillis: 60000
+        minEvictableIdleTimeMillis: 300000
+        validationQuery: SELECT 1 FROM DUAL
+        testWhileIdle: true
+        testOnBorrow: false
+        testOnReturn: false
+        poolPreparedStatements: true
+        maxPoolPreparedStatementPerConnectionSize: 20
+        spring.datasource.filters: stat,wall,log4j
+        connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
+
+#    jpa:
+#      hibernate:
+#        ddl-auto : update
+#    datasource:
+#      platform:
+#      continue-on-error: true
+#      druid:
+#    driver-class-name: org.postgresql.Driver
+#    url: jdbc:postgresql://192.168.220.154:5432/escheduler
+
 
 
 #mybatis
@@ -42,3 +66,26 @@ logging:
   level:
     org.apache.dolphinscheduler.dao: debug
 
+---
+# master settings
+spring:
+  profiles: master
+
+server:
+    port: 5566
+
+---
+# worker settings
+spring:
+  profiles: worker
+
+server:
+  port: 7788
+
+---
+# alert settings
+spring:
+  profiles: alert
+
+server:
+  port: 7789

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml

@@ -5,7 +5,7 @@
         select command.* from t_ds_command command
         join t_ds_process_definition definition on command.process_definition_id = definition.id
         where definition.release_state = 1 AND definition.flag = 1
-        order by update_time asc
+        order by command.update_time asc
         limit 1
     </select>
     <select id="countCommandState" resultType="org.apache.dolphinscheduler.dao.entity.CommandCount">

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java

@@ -58,8 +58,8 @@ public class AccessTokenMapperTest {
         //update
         accessToken.setToken("hello, token");
         int update = accessTokenMapper.updateById(accessToken);
-        Assert.assertEquals(update, 1);
         accessTokenMapper.deleteById(accessToken.getId());
+        Assert.assertEquals(update, 1);
     }
 
     @Test

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java

@@ -109,9 +109,10 @@ public class ProcessInstanceMapperTest {
                 ExecutionStatus.RUNNING_EXEUTION.ordinal(),
                 ExecutionStatus.SUCCESS.ordinal()};
 
-        processInstanceMapper.queryByHostAndStatus(processInstance.getHost(), stateArray);
+        List<ProcessInstance> processInstances = processInstanceMapper.queryByHostAndStatus(null, stateArray);
 
         processInstanceMapper.deleteById(processInstance.getId());
+        Assert.assertNotEquals(processInstances.size(), 0);
     }
 
     @Test

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@@ -39,6 +39,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.dolphinscheduler.common.utils.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.File;
 import java.io.IOException;

dolphinscheduler-server/src/main/resources/master.properties

@@ -15,7 +15,7 @@ master.task.commit.interval=100
 
 
 # only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
-master.max.cpuload.avg=10
+master.max.cpuload.avg=100
 
 # only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
-master.reserved.memory=1
+master.reserved.memory=0.1

dolphinscheduler-ui/.env

@@ -1,5 +1,5 @@
 # 后端接口地址
-API_BASE = http://192.168.xx.xx:12345
+API_BASE = http://127.0.0.1:12345
 
 # 本地开发如需ip访问项目把"#"号去掉
 #DEV_HOST = 192.168.xx.xx

dolphinscheduler-ui/build/webpack.config.dev.js

@@ -88,7 +88,7 @@ const config = merge.smart(baseConfig, {
     historyApiFallback: true,
     disableHostCheck: true,
     proxy: {
-      '/escheduler': {
+      '/dolphinscheduler': {
         target: getEnv('API_BASE', 'http://local.dev:8080/backend'),
         changeOrigin: true
       }

dolphinscheduler-ui/src/js/module/io/index.js

@@ -17,7 +17,7 @@
 
 import io from '~/@fedor/io/dist/io'
 
-const apiPrefix = '/escheduler'
+const apiPrefix = '/dolphinscheduler'
 const reSlashPrefix = /^\/+/
 
 const resolveURL = (url) => {

script/dolphinscheduler-daemon.sh

@@ -44,13 +44,13 @@ if [ "$command" = "api-server" ]; then
   LOG_FILE="-Dlogging.config=conf/apiserver_logback.xml"
   CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
 elif [ "$command" = "master-server" ]; then
-  LOG_FILE="-Dspring.config.location=conf/application_master.properties -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=master -Dlogging.config=conf/master_logback.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.master.MasterServer
 elif [ "$command" = "worker-server" ]; then
-  LOG_FILE="-Dspring.config.location=conf/application_worker.properties -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=worker -Dlogging.config=conf/worker_logback.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
 elif [ "$command" = "alert-server" ]; then
-  LOG_FILE="-Dspring.config.location=conf/application_alert.properties -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=alert -Dlogging.config=conf/alert_logback.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.alert.AlertServer
 elif [ "$command" = "logger-server" ]; then
   CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer