fix too many connection in upgrade or create (#1049)

* rename from DatasourceUserMapper to DataSourceUserMapper

* add unit test in UserMapper and WorkerGroupMapper

* change cn.escheduler to org.apache.dolphinscheduler

* add unit test in UdfFuncMapperTest

* add unit test in UdfFuncMapperTest

* remove DatabaseConfiguration

* add ConnectionFactoryTest

* cal duration in processInstancesList

* change desc to description

* change table name in mysql ddl

* change table name in mysql ddl

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler

* remove log4j-1.2-api and modify AlertMapperTest

* remove log4j-1.2-api

* Add alertDao to spring management

* Add alertDao to spring management

* get SqlSessionFactory from MybatisSqlSessionFactoryBean

* get processDao by DaoFactory

* read druid properties in ConneciontFactory

* read druid properties in ConneciontFactory

* change get alertDao by spring to DaoFactory

* add log4j to resolve #967

* resole verify udf name error and delete udf error

* Determine if principal is empty

* Determine whether the logon user has the right to delete the project

* Fixed an issue that produced attatch file named such as ATT00002.bin

* fix too many connection in upgrade or create

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java

@@ -55,7 +55,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
         ResultSet rs = null;
         Connection conn = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             rs = conn.getMetaData().getTables(null, null, tableName, null);
             if (rs.next()) {
                 return true;
@@ -81,7 +81,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
     public boolean isExistsColumn(String tableName,String columnName) {
         Connection conn = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName);
             if (rs.next()) {
                 return true;

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java

@@ -58,7 +58,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
         PreparedStatement pstmt = null;
         ResultSet resultSet = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             pstmt = conn.prepareStatement("select current_schema()");
             resultSet = pstmt.executeQuery();
             while (resultSet.next()){
@@ -84,7 +84,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
         Connection conn = null;
         ResultSet rs = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
 
             rs = conn.getMetaData().getTables(null, schema, tableName, null);
             if (rs.next()) {
@@ -112,7 +112,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
         Connection conn = null;
         ResultSet rs = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             rs = conn.getMetaData().getColumns(null,schema,tableName,columnName);
             if (rs.next()) {
                 return true;

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dolphinscheduler.dao.upgrade;
 
+import com.alibaba.druid.pool.DruidDataSource;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
 import org.apache.dolphinscheduler.common.utils.SchemaUtils;
@@ -39,6 +41,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
     private static final String T_VERSION_NAME = "t_escheduler_version";
     private static final String T_NEW_VERSION_NAME = "t_ds_version";
     private static final String rootDir = System.getProperty("user.dir");
+    protected static final DruidDataSource dataSource = getDataSource();
     private static final DbType dbType = getCurrentDbType();
 
     @Override
@@ -46,6 +49,19 @@ public abstract class UpgradeDao extends AbstractBaseDao {
 
     }
 
+    /**
+     * get db type
+     * @return
+     */
+    public static DruidDataSource getDataSource(){
+        DruidDataSource dataSource = ConnectionFactory.getDataSource();
+        dataSource.setInitialSize(2);
+        dataSource.setMinIdle(2);
+        dataSource.setMaxActive(2);
+
+        return dataSource;
+    }
+
     /**
      * get db type
      * @return
@@ -61,7 +77,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
     private static DbType getCurrentDbType(){
         Connection conn = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             String name = conn.getMetaData().getDatabaseProductName().toUpperCase();
             return DbType.valueOf(name);
         } catch (Exception e) {
@@ -110,7 +126,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         }
         String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql";
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             conn.setAutoCommit(false);
 
             // Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
@@ -150,7 +166,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         //String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_ddl.sql";
         String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql";
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             // Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
             ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
             Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
@@ -194,7 +210,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         PreparedStatement pstmt = null;
         String version = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             pstmt = conn.prepareStatement(sql);
             rs = pstmt.executeQuery();
 
@@ -231,7 +247,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         Connection conn = null;
         PreparedStatement pstmt = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             conn.setAutoCommit(false);
             // Execute the upgraded dolphinscheduler dml
             ScriptRunner scriptRunner = new ScriptRunner(conn, false, true);
@@ -297,7 +313,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         Connection conn = null;
         PreparedStatement pstmt = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             String dbName = conn.getCatalog();
             logger.info(dbName);
             conn.setAutoCommit(true);
@@ -340,7 +356,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
         PreparedStatement pstmt = null;
         Connection conn = null;
         try {
-            conn = ConnectionFactory.getDataSource().getConnection();
+            conn = dataSource.getConnection();
             pstmt = conn.prepareStatement(upgradeSQL);
             pstmt.setString(1, version);
             pstmt.executeUpdate();

dolphinscheduler-dao/src/main/resources/application.properties

@@ -29,7 +29,7 @@ spring.datasource.timeBetweenConnectErrorMillis=60000
 spring.datasource.minEvictableIdleTimeMillis=300000
 
 #the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
-spring.datasource.validationQuery=SELECT 1 FROM DUAL
+spring.datasource.validationQuery=SELECT 1
 
 #check whether the connection is valid for timeout, in seconds
 spring.datasource.validationQueryTimeout=3
@@ -58,20 +58,25 @@ mybatis-plus.mapper-locations=classpath*:/org.apache.dolphinscheduler.dao.mapper
 mybatis-plus.typeEnumsPackage=org.apache.dolphinscheduler.*.enums
 
 #实体扫描，多个package用逗号或者分号分隔
+#Entity scan, where multiple packages are separated by a comma or semicolon
 mybatis-plus.typeAliasesPackage=org.apache.dolphinscheduler.dao.entity
 
 #主键类型  AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID";
+#Primary key type AUTO:" database ID AUTO ", INPUT:" user INPUT ID", ID_WORKER:" global unique ID (numeric type unique ID)", UUID:" global unique ID UUID";
 mybatis-plus.global-config.db-config.id-type=AUTO
 
 #字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断"
+#Field policy IGNORED:" ignore judgment ",NOT_NULL:" not NULL judgment "),NOT_EMPTY:" not NULL judgment"
 mybatis-plus.global-config.db-config.field-strategy=NOT_NULL
 
 #驼峰下划线转换
+#The hump underline is converted
 mybatis-plus.global-config.db-config.column-underline=true
 mybatis-plus.global-config.db-config.logic-delete-value=-1
 mybatis-plus.global-config.db-config.logic-not-delete-value=0
 mybatis-plus.global-config.db-config.banner=false
 #原生配置
+#The original configuration
 mybatis-plus.configuration.map-underscore-to-camel-case=true
 mybatis-plus.configuration.cache-enabled=false
 mybatis-plus.configuration.call-setters-on-nulls=true