Browse Source

fix too many connection in upgrade or create (#1049)

* rename from DatasourceUserMapper to DataSourceUserMapper

* add unit test in UserMapper and WorkerGroupMapper

* change cn.escheduler to org.apache.dolphinscheduler

* add unit test in UdfFuncMapperTest

* add unit test in UdfFuncMapperTest

* remove DatabaseConfiguration

* add ConnectionFactoryTest

* cal duration in processInstancesList

* change desc to description

* change table name in mysql ddl

* change table name in mysql ddl

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler

* remove log4j-1.2-api and modify AlertMapperTest

* remove log4j-1.2-api

* Add alertDao to spring management

* Add alertDao to spring management

* get SqlSessionFactory from MybatisSqlSessionFactoryBean

* get processDao by DaoFactory

* read druid properties in ConneciontFactory

* read druid properties in ConneciontFactory

* change get alertDao by spring to DaoFactory

* add log4j to resolve #967

* resole verify udf name error and delete udf error

* Determine if principal is empty

* Determine whether the logon user has the right to delete the project

* Fixed an issue that produced attatch file named such as ATT00002.bin

* fix too many connection in upgrade or create
pull/2/head
lgcareer 5 years ago committed by qiaozhanwei
parent
commit
5dc8beebff
  1. 4
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java
  2. 6
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java
  3. 30
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
  4. 7
      dolphinscheduler-dao/src/main/resources/application.properties

4
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java

@ -55,7 +55,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
ResultSet rs = null; ResultSet rs = null;
Connection conn = null; Connection conn = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
rs = conn.getMetaData().getTables(null, null, tableName, null); rs = conn.getMetaData().getTables(null, null, tableName, null);
if (rs.next()) { if (rs.next()) {
return true; return true;
@ -81,7 +81,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
public boolean isExistsColumn(String tableName,String columnName) { public boolean isExistsColumn(String tableName,String columnName) {
Connection conn = null; Connection conn = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName); ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName);
if (rs.next()) { if (rs.next()) {
return true; return true;

6
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java

@ -58,7 +58,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
PreparedStatement pstmt = null; PreparedStatement pstmt = null;
ResultSet resultSet = null; ResultSet resultSet = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
pstmt = conn.prepareStatement("select current_schema()"); pstmt = conn.prepareStatement("select current_schema()");
resultSet = pstmt.executeQuery(); resultSet = pstmt.executeQuery();
while (resultSet.next()){ while (resultSet.next()){
@ -84,7 +84,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
Connection conn = null; Connection conn = null;
ResultSet rs = null; ResultSet rs = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
rs = conn.getMetaData().getTables(null, schema, tableName, null); rs = conn.getMetaData().getTables(null, schema, tableName, null);
if (rs.next()) { if (rs.next()) {
@ -112,7 +112,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
Connection conn = null; Connection conn = null;
ResultSet rs = null; ResultSet rs = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
rs = conn.getMetaData().getColumns(null,schema,tableName,columnName); rs = conn.getMetaData().getColumns(null,schema,tableName,columnName);
if (rs.next()) { if (rs.next()) {
return true; return true;

30
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -16,6 +16,8 @@
*/ */
package org.apache.dolphinscheduler.dao.upgrade; package org.apache.dolphinscheduler.dao.upgrade;
import com.alibaba.druid.pool.DruidDataSource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.SchemaUtils; import org.apache.dolphinscheduler.common.utils.SchemaUtils;
@ -39,6 +41,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
private static final String T_VERSION_NAME = "t_escheduler_version"; private static final String T_VERSION_NAME = "t_escheduler_version";
private static final String T_NEW_VERSION_NAME = "t_ds_version"; private static final String T_NEW_VERSION_NAME = "t_ds_version";
private static final String rootDir = System.getProperty("user.dir"); private static final String rootDir = System.getProperty("user.dir");
protected static final DruidDataSource dataSource = getDataSource();
private static final DbType dbType = getCurrentDbType(); private static final DbType dbType = getCurrentDbType();
@Override @Override
@ -46,6 +49,19 @@ public abstract class UpgradeDao extends AbstractBaseDao {
} }
/**
* get db type
* @return
*/
public static DruidDataSource getDataSource(){
DruidDataSource dataSource = ConnectionFactory.getDataSource();
dataSource.setInitialSize(2);
dataSource.setMinIdle(2);
dataSource.setMaxActive(2);
return dataSource;
}
/** /**
* get db type * get db type
* @return * @return
@ -61,7 +77,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
private static DbType getCurrentDbType(){ private static DbType getCurrentDbType(){
Connection conn = null; Connection conn = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
String name = conn.getMetaData().getDatabaseProductName().toUpperCase(); String name = conn.getMetaData().getDatabaseProductName().toUpperCase();
return DbType.valueOf(name); return DbType.valueOf(name);
} catch (Exception e) { } catch (Exception e) {
@ -110,7 +126,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
} }
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql"; String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql";
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
conn.setAutoCommit(false); conn.setAutoCommit(false);
// Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler // Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
@ -150,7 +166,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
//String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_ddl.sql"; //String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_ddl.sql";
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql"; String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql";
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
// Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler // Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true); ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
@ -194,7 +210,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
PreparedStatement pstmt = null; PreparedStatement pstmt = null;
String version = null; String version = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
pstmt = conn.prepareStatement(sql); pstmt = conn.prepareStatement(sql);
rs = pstmt.executeQuery(); rs = pstmt.executeQuery();
@ -231,7 +247,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
Connection conn = null; Connection conn = null;
PreparedStatement pstmt = null; PreparedStatement pstmt = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
conn.setAutoCommit(false); conn.setAutoCommit(false);
// Execute the upgraded dolphinscheduler dml // Execute the upgraded dolphinscheduler dml
ScriptRunner scriptRunner = new ScriptRunner(conn, false, true); ScriptRunner scriptRunner = new ScriptRunner(conn, false, true);
@ -297,7 +313,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
Connection conn = null; Connection conn = null;
PreparedStatement pstmt = null; PreparedStatement pstmt = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
String dbName = conn.getCatalog(); String dbName = conn.getCatalog();
logger.info(dbName); logger.info(dbName);
conn.setAutoCommit(true); conn.setAutoCommit(true);
@ -340,7 +356,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
PreparedStatement pstmt = null; PreparedStatement pstmt = null;
Connection conn = null; Connection conn = null;
try { try {
conn = ConnectionFactory.getDataSource().getConnection(); conn = dataSource.getConnection();
pstmt = conn.prepareStatement(upgradeSQL); pstmt = conn.prepareStatement(upgradeSQL);
pstmt.setString(1, version); pstmt.setString(1, version);
pstmt.executeUpdate(); pstmt.executeUpdate();

7
dolphinscheduler-dao/src/main/resources/application.properties

@ -29,7 +29,7 @@ spring.datasource.timeBetweenConnectErrorMillis=60000
spring.datasource.minEvictableIdleTimeMillis=300000 spring.datasource.minEvictableIdleTimeMillis=300000
#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. #the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
spring.datasource.validationQuery=SELECT 1 FROM DUAL spring.datasource.validationQuery=SELECT 1
#check whether the connection is valid for timeout, in seconds #check whether the connection is valid for timeout, in seconds
spring.datasource.validationQueryTimeout=3 spring.datasource.validationQueryTimeout=3
@ -58,20 +58,25 @@ mybatis-plus.mapper-locations=classpath*:/org.apache.dolphinscheduler.dao.mapper
mybatis-plus.typeEnumsPackage=org.apache.dolphinscheduler.*.enums mybatis-plus.typeEnumsPackage=org.apache.dolphinscheduler.*.enums
#实体扫描,多个package用逗号或者分号分隔 #实体扫描,多个package用逗号或者分号分隔
#Entity scan, where multiple packages are separated by a comma or semicolon
mybatis-plus.typeAliasesPackage=org.apache.dolphinscheduler.dao.entity mybatis-plus.typeAliasesPackage=org.apache.dolphinscheduler.dao.entity
#主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID"; #主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID";
#Primary key type AUTO:" database ID AUTO ", INPUT:" user INPUT ID", ID_WORKER:" global unique ID (numeric type unique ID)", UUID:" global unique ID UUID";
mybatis-plus.global-config.db-config.id-type=AUTO mybatis-plus.global-config.db-config.id-type=AUTO
#字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断" #字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断"
#Field policy IGNORED:" ignore judgment ",NOT_NULL:" not NULL judgment "),NOT_EMPTY:" not NULL judgment"
mybatis-plus.global-config.db-config.field-strategy=NOT_NULL mybatis-plus.global-config.db-config.field-strategy=NOT_NULL
#驼峰下划线转换 #驼峰下划线转换
#The hump underline is converted
mybatis-plus.global-config.db-config.column-underline=true mybatis-plus.global-config.db-config.column-underline=true
mybatis-plus.global-config.db-config.logic-delete-value=-1 mybatis-plus.global-config.db-config.logic-delete-value=-1
mybatis-plus.global-config.db-config.logic-not-delete-value=0 mybatis-plus.global-config.db-config.logic-not-delete-value=0
mybatis-plus.global-config.db-config.banner=false mybatis-plus.global-config.db-config.banner=false
#原生配置 #原生配置
#The original configuration
mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.configuration.map-underscore-to-camel-case=true
mybatis-plus.configuration.cache-enabled=false mybatis-plus.configuration.cache-enabled=false
mybatis-plus.configuration.call-setters-on-nulls=true mybatis-plus.configuration.call-setters-on-nulls=true

Loading…
Cancel
Save