From 57080c96e868f0c0eb67272fa3e837d50cc1adc7 Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Thu, 10 Oct 2019 17:12:04 +0800 Subject: [PATCH] update application.yml to application.properties (#980) * update english documents * refactor zk client * update documents * update zkclient * update zkclient * update documents * add architecture-design * change i18n * update i18n * update english documents * add architecture-design * update english documents * update en-US documents * add architecture-design * update demo site * add mybatis plus model * modify mybatisplus * modify mybatisplus * change interface by mybatisplus * add unit test * refactor dao interface. * add unit test for dao... * add unit test for dao... * add unit test for dao... * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update some dao bugs * update for some bugs * update some bugs * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update * update * add multiply settings for application.yml * add multiply settings for application.yml * revert * update configuration settings in task record dao... * change application_master to application-master * change application_master to application-master * update application.yml to application.properties * revert --- ...roperties => application-alert.properties} | 0 ....properties => application-api.properties} | 0 .../dolphinscheduler/dao/MonitorDBDao.java | 25 +++-- .../dolphinscheduler/dao/ProcessDao.java | 2 +- .../dolphinscheduler/dao/TaskRecordDao.java | 7 +- .../dao/config/YmlConfig.java | 67 -------------- .../dao/datasource/ConnectionFactory.java | 29 ++++-- .../dao/mapper/CommandMapper.java | 4 - .../src/main/resources/application.yml | 91 ------------------- ...operties => application-master.properties} | 0 ...operties => application-worker.properties} | 0 script/dolphinscheduler-daemon.sh | 6 +- 12 files changed, 40 insertions(+), 191 deletions(-) rename dolphinscheduler-alert/src/main/resources/{application_alert.properties => application-alert.properties} (100%) rename dolphinscheduler-api/src/main/resources/{application.properties => application-api.properties} (100%) delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/config/YmlConfig.java rename dolphinscheduler-server/src/main/resources/{application_master.properties => application-master.properties} (100%) rename dolphinscheduler-server/src/main/resources/{application_worker.properties => application-worker.properties} (100%) diff --git a/dolphinscheduler-alert/src/main/resources/application_alert.properties b/dolphinscheduler-alert/src/main/resources/application-alert.properties similarity index 100% rename from dolphinscheduler-alert/src/main/resources/application_alert.properties rename to dolphinscheduler-alert/src/main/resources/application-alert.properties diff --git a/dolphinscheduler-api/src/main/resources/application.properties b/dolphinscheduler-api/src/main/resources/application-api.properties similarity index 100% rename from dolphinscheduler-api/src/main/resources/application.properties rename to dolphinscheduler-api/src/main/resources/application-api.properties diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 6e2023e52f..b9a99eec31 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.dao.config.YmlConfig; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; @@ -42,16 +41,16 @@ public class MonitorDBDao { /** * 加载配置文件 */ -// private static Configuration conf; + private static Configuration conf; -// static { -// try { -// conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES); -// }catch (ConfigurationException e){ -// logger.error("load configuration excetpion",e); -// System.exit(1); -// } -// } + static { + try { + conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES); + }catch (ConfigurationException e){ + logger.error("load configuration excetpion",e); + System.exit(1); + } + } /** @@ -59,9 +58,9 @@ public class MonitorDBDao { * @return */ private static Connection getConn() { - String url = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_URL); - String username = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_USERNAME); - String password = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_PASSWORD); + String url = conf.getString(Constants.SPRING_DATASOURCE_URL); + String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME); + String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD); Connection conn = null; try { //classloader,load driver diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java index 04fe11f23a..5f442d38f3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java @@ -231,7 +231,7 @@ public class ProcessDao extends AbstractBaseDao { JSONObject tempObj; int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); - List commands = commandMapper.getAll(null); + List commands = commandMapper.selectList(null); //遍历所有命令 for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index 2c522a2183..7981c41297 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -42,10 +42,8 @@ public class TaskRecordDao { private static Logger logger = LoggerFactory.getLogger(TaskRecordDao.class.getName()); - - /** - * load conf file + * 加载配置文件 */ private static Configuration conf; @@ -58,7 +56,6 @@ public class TaskRecordDao { } } - /** * get task record flag * @return @@ -71,7 +68,7 @@ public class TaskRecordDao { * @return */ private static Connection getConn() { - if(!conf.getBoolean(Constants.TASK_RECORD_FLAG)){ + if(!getTaskRecordFlag()){ return null; } String driver = "com.mysql.jdbc.Driver"; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/config/YmlConfig.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/config/YmlConfig.java deleted file mode 100644 index a2e0f65651..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/config/YmlConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.config; - - - -import org.yaml.snakeyaml.*; - -import java.io.InputStream; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; - -public class YmlConfig { - - public static Map allMap=new HashMap(); - static { - Yaml yaml = new Yaml(); - InputStream inputStream = YmlConfig.class.getResourceAsStream("/application.yml"); - Iterator result = yaml.loadAll(inputStream).iterator(); - while(result.hasNext()){ - Map map=(Map)result.next(); - iteratorYml( map,null); - } - } - - public static void iteratorYml(Map map,String key){ - Iterator iterator = map.entrySet().iterator(); - while(iterator.hasNext()){ - Map.Entry entry = (Map.Entry) iterator.next(); - Object key2 = entry.getKey(); - Object value = entry.getValue(); - if(value instanceof LinkedHashMap){ - if(key==null){ - iteratorYml((Map)value,key2.toString()); - }else{ - iteratorYml((Map)value,key+"."+key2.toString()); - } - } - if(value instanceof String){ - if(key==null){ - allMap.put(key2.toString(), value.toString()); - } - if(key!=null){ - allMap.put(key+"."+key2.toString(), value.toString()); - } - } - } - - } - -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java index b9e9aecd15..aa1d4a77c5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.dao.datasource; import com.alibaba.druid.pool.DruidDataSource; -import org.apache.dolphinscheduler.dao.config.YmlConfig; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.dolphinscheduler.common.Constants; import org.apache.ibatis.mapping.Environment; import org.apache.ibatis.session.Configuration; import org.apache.ibatis.session.SqlSession; @@ -30,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.sql.DataSource; -import java.util.Map; /** @@ -41,16 +42,30 @@ public class ConnectionFactory { private static SqlSessionFactory sqlSessionFactory; + /** + * 加载配置文件 + */ + private static org.apache.commons.configuration.Configuration conf; + + static { + try { + conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES); + }catch (ConfigurationException e){ + logger.error("load configuration excetpion",e); + System.exit(1); + } + } + /** * get the data source */ public static DruidDataSource getDataSource() { DruidDataSource druidDataSource = new DruidDataSource(); - Map allMap = YmlConfig.allMap; - druidDataSource.setDriverClassName(allMap.get("spring.datasource.driver-class-name")); - druidDataSource.setUrl(allMap.get("spring.datasource.url")); - druidDataSource.setUsername(allMap.get("spring.datasource.username")); - druidDataSource.setPassword(allMap.get("spring.datasource.password")); +// Map allMap = YmlConfig.allMap; + druidDataSource.setDriverClassName(conf.getString("spring.datasource.driver-class-name")); + druidDataSource.setUrl(conf.getString("spring.datasource.url")); + druidDataSource.setUsername(conf.getString("spring.datasource.username")); + druidDataSource.setPassword(conf.getString("spring.datasource.password")); druidDataSource.setInitialSize(5); druidDataSource.setMinIdle(5); druidDataSource.setMaxActive(20); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index 9747dd8c7d..e727afaaf1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -30,10 +30,6 @@ import java.util.List; public interface CommandMapper extends BaseMapper { - - @Select("select * from t_ds_command ${ew.customSqlSegment}") - List getAll(@Param(Constants.WRAPPER) Wrapper wrapper); - Command getOneToRun(); List countCommandState( diff --git a/dolphinscheduler-dao/src/main/resources/application.yml b/dolphinscheduler-dao/src/main/resources/application.yml index 6c3c258e3a..e69de29bb2 100644 --- a/dolphinscheduler-dao/src/main/resources/application.yml +++ b/dolphinscheduler-dao/src/main/resources/application.yml @@ -1,91 +0,0 @@ -# mysql -spring: - - datasource: - url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - username: root - password: root@123 - # - type: com.alibaba.druid.pool.DruidDataSource - driver-class-name: com.mysql.jdbc.Driver - initialSize: 5 - minIdle: 5 - maxActive: 20 - maxWait: 60000 - timeBetweenEvictionRunsMillis: 60000 - minEvictableIdleTimeMillis: 300000 - validationQuery: SELECT 1 FROM DUAL - testWhileIdle: true - testOnBorrow: false - testOnReturn: false - poolPreparedStatements: true - maxPoolPreparedStatementPerConnectionSize: 20 - spring.datasource.filters: stat,wall,log4j - connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - -# jpa: -# hibernate: -# ddl-auto : update -# datasource: -# platform: -# continue-on-error: true -# druid: -# driver-class-name: org.postgresql.Driver -# url: jdbc:postgresql://192.168.220.154:5432/escheduler - - - -#mybatis -mybatis-plus: - mapper-locations: classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml - typeEnumsPackage: org.apache.dolphinscheduler.*.enums - #实体扫描，多个package用逗号或者分号分隔 - typeAliasesPackage: org.apache.dolphinscheduler.dao.entity - global-config: - #数据库相关配置 - db-config: - #主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID"; - id-type: AUTO - #字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断" - field-strategy: NOT_NULL - #驼峰下划线转换 - column-underline: true - logic-delete-value: -1 - logic-not-delete-value: 0 - banner: false - #原生配置 - configuration: - map-underscore-to-camel-case: true - cache-enabled: false - call-setters-on-nulls: true - jdbc-type-for-null: 'null' - - -# Logger Config -logging: - level: - org.apache.dolphinscheduler.dao: debug - ---- -# master settings -spring: - profiles: master - -server: - port: 5566 - ---- -# worker settings -spring: - profiles: worker - -server: - port: 7788 - ---- -# alert settings -spring: - profiles: alert - -server: - port: 7789 \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/application_master.properties b/dolphinscheduler-server/src/main/resources/application-master.properties similarity index 100% rename from dolphinscheduler-server/src/main/resources/application_master.properties rename to dolphinscheduler-server/src/main/resources/application-master.properties diff --git a/dolphinscheduler-server/src/main/resources/application_worker.properties b/dolphinscheduler-server/src/main/resources/application-worker.properties similarity index 100% rename from dolphinscheduler-server/src/main/resources/application_worker.properties rename to dolphinscheduler-server/src/main/resources/application-worker.properties diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 4c7c675920..45898a013b 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -44,13 +44,13 @@ if [ "$command" = "api-server" ]; then LOG_FILE="-Dlogging.config=conf/apiserver_logback.xml" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dspring.profiles.active=master -Dlogging.config=conf/master_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dspring.profiles.active=worker -Dlogging.config=conf/worker_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dspring.profiles.active=alert -Dlogging.config=conf/alert_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dspring.profiles.active=alert -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer