Query return number In SQL should be configurable. (#5896)

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.task.sql;
 
 import org.apache.dolphinscheduler.common.process.ResourceInfo;
@@ -24,7 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 /**
- * Sql/Hql parameter
+ * Sql/Hql parameter.
  */
 public class SqlParameters extends AbstractParameters {
     /**
@@ -103,6 +104,19 @@ public class SqlParameters extends AbstractParameters {
      */
     private String receiversCc;
 
+    /**
+     * query result limit
+     */
+    private int limit;
+
+    public int getLimit() {
+        return limit;
+    }
+
+    public void setLimit(int limit) {
+        this.limit = limit;
+    }
+
     public String getType() {
         return type;
     }
@@ -226,21 +240,22 @@ public class SqlParameters extends AbstractParameters {
 
     @Override
     public String toString() {
-        return "SqlParameters{" +
+        return "SqlParameters{"
-                "type='" + type + '\'' +
+                + "type='" + type + '\''
-                ", datasource=" + datasource +
+                + ", datasource=" + datasource
-                ", sql='" + sql + '\'' +
+                + ", sql='" + sql + '\''
-                ", sqlType=" + sqlType +
+                + ", sqlType=" + sqlType
-                ", sendEmail=" + sendEmail +
+                + ", sendEmail=" + sendEmail
-                ", displayRows=" + displayRows +
+                + ", displayRows=" + displayRows
-                ", udfs='" + udfs + '\'' +
+                + ", limit=" + limit
-                ", showType='" + showType + '\'' +
+                + ", udfs='" + udfs + '\''
-                ", connParams='" + connParams + '\'' +
+                + ", showType='" + showType + '\''
-                ", title='" + title + '\'' +
+                + ", connParams='" + connParams + '\''
-                ", receivers='" + receivers + '\'' +
+                + ", title='" + title + '\''
-                ", receiversCc='" + receiversCc + '\'' +
+                + ", receivers='" + receivers + '\''
-                ", preStatements=" + preStatements +
+                + ", receiversCc='" + receiversCc + '\''
-                ", postStatements=" + postStatements +
+                + ", preStatements=" + preStatements
-                '}';
+                + ", postStatements=" + postStatements
+                + '}';
     }
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java

@@ -32,6 +32,7 @@ public class SqlParametersTest {
     private final int sqlType = 0;
     private final Boolean sendEmail = true;
     private final int displayRows = 10;
+    private final int limit = 0;
     private final String showType = "TABLE";
     private final String title = "sql test";
 
@@ -49,6 +50,7 @@ public class SqlParametersTest {
         sqlParameters.setDisplayRows(displayRows);
         sqlParameters.setShowType(showType);
         sqlParameters.setTitle(title);
+        sqlParameters.setLimit(limit);
 
         Assert.assertEquals(type, sqlParameters.getType());
         Assert.assertEquals(sql, sqlParameters.getSql());
@@ -59,6 +61,7 @@ public class SqlParametersTest {
         Assert.assertEquals(displayRows, sqlParameters.getDisplayRows());
         Assert.assertEquals(showType, sqlParameters.getShowType());
         Assert.assertEquals(title, sqlParameters.getTitle());
+        Assert.assertEquals(limit, sqlParameters.getLimit());
 
         Assert.assertTrue(sqlParameters.checkParameters());
     }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@@ -16,14 +16,9 @@
  */
 package org.apache.dolphinscheduler.server.worker.task.sql;
 
-import static org.apache.dolphinscheduler.common.Constants.COMMA;
+import com.alibaba.fastjson.JSONArray;
-import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF;
+import com.alibaba.fastjson.JSONObject;
-import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
+import org.apache.commons.lang.StringUtils;
-import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
-import static org.apache.dolphinscheduler.common.Constants.STATUS;
-import static org.apache.dolphinscheduler.common.Constants.USER;
-import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
-
 import org.apache.dolphinscheduler.alert.utils.MailUtils;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -35,11 +30,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
 import org.apache.dolphinscheduler.common.task.sql.SqlBinds;
 import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
 import org.apache.dolphinscheduler.common.task.sql.SqlType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.dolphinscheduler.common.utils.*;
-import org.apache.dolphinscheduler.common.utils.CommonUtils;
-import org.apache.dolphinscheduler.common.utils.EnumUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.dao.AlertDao;
 import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
 import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
@@ -50,30 +41,16 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils;
 import org.apache.dolphinscheduler.server.utils.UDFUtils;
 import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.slf4j.Logger;
 
-import org.apache.commons.lang.StringUtils;
+import java.sql.*;
-
+import java.util.*;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import org.slf4j.Logger;
+import static org.apache.dolphinscheduler.common.Constants.*;
-
+import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
 /**
  * sql task
  */
@@ -97,11 +74,6 @@ public class SqlTask extends AbstractTask {
      */
     private TaskExecutionContext taskExecutionContext;
 
-    /**
-     * default query sql limit
-     */
-    private static final int LIMIT = 10000;
-
     public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) {
         super(taskExecutionContext, logger);
 
@@ -124,14 +96,15 @@ public class SqlTask extends AbstractTask {
         Thread.currentThread().setName(threadLoggerInfoName);
 
         logger.info("Full sql parameters: {}", sqlParameters);
-        logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}",
+        logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}, query max result limit : {}",
                 sqlParameters.getType(),
                 sqlParameters.getDatasource(),
                 sqlParameters.getSql(),
                 sqlParameters.getLocalParams(),
                 sqlParameters.getUdfs(),
                 sqlParameters.getShowType(),
-                sqlParameters.getConnParams());
+                sqlParameters.getConnParams(),
+                sqlParameters.getLimit());
         try {
             SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext();
             // load class
@@ -282,7 +255,7 @@ public class SqlTask extends AbstractTask {
 
         int rowCount = 0;
 
-        while (rowCount < LIMIT && resultSet.next()) {
+        while (rowCount < sqlParameters.getLimit() && resultSet.next()) {
             JSONObject mapOfColValues = new JSONObject(true);
             for (int i = 1; i <= num; i++) {
                 mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i));

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue

@@ -46,6 +46,13 @@
         </div>
       </div>
     </m-list-box>
+    <m-list-box v-show="sqlType === '0'">
+      <div slot="text"><strong class='requiredIcon'>*</strong>{{$t('Max Numbers Return')}}</div>
+      <div slot="content">
+        <x-input type="input" :disabled="isDetails" size="medium" v-model="limit" :placeholder="$t('Max Numbers Return placeholder')">
+        </x-input>
+      </div>
+    </m-list-box>
     <template v-if="sqlType==0 && sendEmail">
       <m-list-box>
         <div slot="text">{{$t('Show Type')}}</div>
@@ -187,6 +194,8 @@
         sendEmail: false,
         // Display rows
         displayRows: 10,
+        // Max returned rows
+        limit: 10000,
         // Email title
         title: '',
         // Form/attachment
@@ -209,6 +218,13 @@
       createNodeId: Number
     },
     methods: {
+      /**
+       * limit should't be empty;limit should be a non-negative number str;
+       * limit should be a number smaller or equal than Integer.MAX_VALUE in java.
+       */
+      isLimitInvalid () {
+        return !this.limit || !/^(0|[1-9]\d*)$/.test(this.limit) || parseInt(this.limit, 10) > 2147483647
+      },
       /**
        * return sqlType
        */
@@ -270,6 +286,10 @@
           this.$message.warning(`${i18n.$t('Mail subject required')}`)
           return false
         }
+        if (this.sqlType === '0' && this.isLimitInvalid()) {
+          this.$message.warning(`${i18n.$t('Max Numbers Return required')}`)
+          return false
+        }
         if (this.sqlType==0 && this.sendEmail && !this.receivers.length) {
           this.$message.warning(`${i18n.$t('Recipient required')}`)
           return false
@@ -313,6 +333,7 @@
           sqlType: this.sqlType,
           sendEmail: this.sendEmail,
           displayRows: this.displayRows,
+          limit: this.limit,
           title: this.title,
           receivers: this.receivers.join(','),
           receiversCc: this.receiversCc.join(','),
@@ -390,6 +411,7 @@
           sqlType: this.sqlType,
           sendEmail: this.sendEmail,
           displayRows: this.displayRows,
+          limit: this.limit,
           title: this.title,
           receivers: this.receivers.join(','),
           receiversCc: this.receiversCc.join(','),
@@ -452,6 +474,7 @@
         this.sqlType = o.params.sqlType
         this.sendEmail = o.params.sendEmail || false
         this.displayRows = o.params.displayRows || 10
+        this.limit = o.params.limit || 10000
         this.connParams = o.params.connParams || ''
         this.localParams = o.params.localParams || []
         if(o.params.showType == '') {
@@ -498,6 +521,7 @@
           sqlType: this.sqlType,
           sendEmail: this.sendEmail,
           displayRows: this.displayRows,
+          limit: this.limit,
           title: this.title,
           receivers: this.receivers.join(','),
           receiversCc: this.receiversCc.join(','),

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -129,6 +129,9 @@ export default {
   'Send Email': 'Send Email',
   'Log display': 'Log display',
   'rows of result': 'rows of result',
+  'Max Numbers Return': 'Number of records to return',
+  'Max Numbers Return placeholder': 'Default is 10000, a large value may cause high pressure on the memory',
+  'Max Numbers Return required': 'Number of records to return parameter must be a number in the range of 0 - 2147483647',
   'Show Type': 'Show Type',
   Title: 'Title',
   'Please enter the title of email': 'Please enter the title of email',

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -129,6 +129,9 @@ export default {
   'Send Email': '发送邮件',
   'Log display': '日志显示',
   'rows of result': '行查询结果',
+  'Max Numbers Return': '返回的记录行数',
+  'Max Numbers Return placeholder': '默认值10000,如果值过大可能会对内存造成较大压力',
+  'Max Numbers Return required': '返回的记录行数值必须是一个在0-2147483647范围内的整数',
   'Show Type': '呈现方式',
   Title: '主题',
   'Please enter the title of email': '请输入邮件主题',