Query return number In SQL should be configurable. (#5896)

3 years ago · 79e7c4f051
6 changed files with 78 additions and 57 deletions
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
@ -14,6 +14,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
+
 package org.apache.dolphinscheduler.common.task.sql;

 import org.apache.dolphinscheduler.common.process.ResourceInfo;
@ -24,7 +25,7 @@ import java.util.ArrayList;
 import java.util.List;

 /**
- * Sql/Hql parameter
+ * Sql/Hql parameter.
 */
 public class SqlParameters extends AbstractParameters {
    /**
@ -103,6 +104,19 @@ public class SqlParameters extends AbstractParameters {
     */
    private String receiversCc;

+    /**
+     * query result limit
+     */
+    private int limit;
+
+    public int getLimit() {
+        return limit;
+    }
+
+    public void setLimit(int limit) {
+        this.limit = limit;
+    }
+
    public String getType() {
        return type;
    }
@ -226,21 +240,22 @@ public class SqlParameters extends AbstractParameters {

    @Override
    public String toString() {
-        return "SqlParameters{" +
-                "type='" + type + '\'' +
-                ", datasource=" + datasource +
-                ", sql='" + sql + '\'' +
-                ", sqlType=" + sqlType +
-                ", sendEmail=" + sendEmail +
-                ", displayRows=" + displayRows +
-                ", udfs='" + udfs + '\'' +
-                ", showType='" + showType + '\'' +
-                ", connParams='" + connParams + '\'' +
-                ", title='" + title + '\'' +
-                ", receivers='" + receivers + '\'' +
-                ", receiversCc='" + receiversCc + '\'' +
-                ", preStatements=" + preStatements +
-                ", postStatements=" + postStatements +
-                '}';
+        return "SqlParameters{"
+                + "type='" + type + '\''
+                + ", datasource=" + datasource
+                + ", sql='" + sql + '\''
+                + ", sqlType=" + sqlType
+                + ", sendEmail=" + sendEmail
+                + ", displayRows=" + displayRows
+                + ", limit=" + limit
+                + ", udfs='" + udfs + '\''
+                + ", showType='" + showType + '\''
+                + ", connParams='" + connParams + '\''
+                + ", title='" + title + '\''
+                + ", receivers='" + receivers + '\''
+                + ", receiversCc='" + receiversCc + '\''
+                + ", preStatements=" + preStatements
+                + ", postStatements=" + postStatements
+                + '}';
    }
 }
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
@ -32,6 +32,7 @@ public class SqlParametersTest {
    private final int sqlType = 0;
    private final Boolean sendEmail = true;
    private final int displayRows = 10;
+    private final int limit = 0;
    private final String showType = "TABLE";
    private final String title = "sql test";

@ -49,6 +50,7 @@ public class SqlParametersTest {
        sqlParameters.setDisplayRows(displayRows);
        sqlParameters.setShowType(showType);
        sqlParameters.setTitle(title);
+        sqlParameters.setLimit(limit);

        Assert.assertEquals(type, sqlParameters.getType());
        Assert.assertEquals(sql, sqlParameters.getSql());
@ -59,6 +61,7 @@ public class SqlParametersTest {
        Assert.assertEquals(displayRows, sqlParameters.getDisplayRows());
        Assert.assertEquals(showType, sqlParameters.getShowType());
        Assert.assertEquals(title, sqlParameters.getTitle());
+        Assert.assertEquals(limit, sqlParameters.getLimit());

        Assert.assertTrue(sqlParameters.checkParameters());
    }
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
@ -16,14 +16,9 @@
 */
 package org.apache.dolphinscheduler.server.worker.task.sql;

-import static org.apache.dolphinscheduler.common.Constants.COMMA;
-import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF;
-import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
-import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
-import static org.apache.dolphinscheduler.common.Constants.STATUS;
-import static org.apache.dolphinscheduler.common.Constants.USER;
-import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
-
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.lang.StringUtils;
 import org.apache.dolphinscheduler.alert.utils.MailUtils;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
@ -35,11 +30,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
 import org.apache.dolphinscheduler.common.task.sql.SqlBinds;
 import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
 import org.apache.dolphinscheduler.common.task.sql.SqlType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.CommonUtils;
-import org.apache.dolphinscheduler.common.utils.EnumUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import org.apache.dolphinscheduler.common.utils.*;
 import org.apache.dolphinscheduler.dao.AlertDao;
 import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
 import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
@ -50,30 +41,16 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils;
 import org.apache.dolphinscheduler.server.utils.UDFUtils;
 import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.slf4j.Logger;

-import org.apache.commons.lang.StringUtils;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
+import java.sql.*;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

-import org.slf4j.Logger;
-
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import static org.apache.dolphinscheduler.common.Constants.*;
+import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
 /**
 * sql task
 */
@ -97,11 +74,6 @@ public class SqlTask extends AbstractTask {
     */
    private TaskExecutionContext taskExecutionContext;

-    /**
-     * default query sql limit
-     */
-    private static final int LIMIT = 10000;
-
    public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) {
        super(taskExecutionContext, logger);

@ -124,14 +96,15 @@ public class SqlTask extends AbstractTask {
        Thread.currentThread().setName(threadLoggerInfoName);

        logger.info("Full sql parameters: {}", sqlParameters);
-        logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}",
+        logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}, query max result limit : {}",
                sqlParameters.getType(),
                sqlParameters.getDatasource(),
                sqlParameters.getSql(),
                sqlParameters.getLocalParams(),
                sqlParameters.getUdfs(),
                sqlParameters.getShowType(),
-                sqlParameters.getConnParams());
+                sqlParameters.getConnParams(),
+                sqlParameters.getLimit());
        try {
            SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext();
            // load class
@ -282,7 +255,7 @@ public class SqlTask extends AbstractTask {

        int rowCount = 0;

-        while (rowCount < LIMIT && resultSet.next()) {
+        while (rowCount < sqlParameters.getLimit() && resultSet.next()) {
            JSONObject mapOfColValues = new JSONObject(true);
            for (int i = 1; i <= num; i++) {
                mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i));
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
@ -46,6 +46,13 @@
        </div>
      </div>
    </m-list-box>
+    <m-list-box v-show="sqlType === '0'">
+      <div slot="text"><strong class='requiredIcon'>*</strong>{{$t('Max Numbers Return')}}</div>
+      <div slot="content">
+        <x-input type="input" :disabled="isDetails" size="medium" v-model="limit" :placeholder="$t('Max Numbers Return placeholder')">
+        </x-input>
+      </div>
+    </m-list-box>
    <template v-if="sqlType==0 && sendEmail">
      <m-list-box>
        <div slot="text">{{$t('Show Type')}}</div>
@ -187,6 +194,8 @@
        sendEmail: false,
        // Display rows
        displayRows: 10,
+        // Max returned rows
+        limit: 10000,
        // Email title
        title: '',
        // Form/attachment
@ -209,6 +218,13 @@
      createNodeId: Number
    },
    methods: {
+      /**
+       * limit should't be empty;limit should be a non-negative number str;
+       * limit should be a number smaller or equal than Integer.MAX_VALUE in java.
+       */
+      isLimitInvalid () {
+        return !this.limit || !/^(0|[1-9]\d*)$/.test(this.limit) || parseInt(this.limit, 10) > 2147483647
+      },
      /**
       * return sqlType
       */
@ -270,6 +286,10 @@
          this.$message.warning(`${i18n.$t('Mail subject required')}`)
          return false
        }
+        if (this.sqlType === '0' && this.isLimitInvalid()) {
+          this.$message.warning(`${i18n.$t('Max Numbers Return required')}`)
+          return false
+        }
        if (this.sqlType==0 && this.sendEmail && !this.receivers.length) {
          this.$message.warning(`${i18n.$t('Recipient required')}`)
          return false
@ -313,6 +333,7 @@
          sqlType: this.sqlType,
          sendEmail: this.sendEmail,
          displayRows: this.displayRows,
+          limit: this.limit,
          title: this.title,
          receivers: this.receivers.join(','),
          receiversCc: this.receiversCc.join(','),
@ -390,6 +411,7 @@
          sqlType: this.sqlType,
          sendEmail: this.sendEmail,
          displayRows: this.displayRows,
+          limit: this.limit,
          title: this.title,
          receivers: this.receivers.join(','),
          receiversCc: this.receiversCc.join(','),
@ -452,6 +474,7 @@
        this.sqlType = o.params.sqlType
        this.sendEmail = o.params.sendEmail || false
        this.displayRows = o.params.displayRows || 10
+        this.limit = o.params.limit || 10000
        this.connParams = o.params.connParams || ''
        this.localParams = o.params.localParams || []
        if(o.params.showType == '') {
@ -498,6 +521,7 @@
          sqlType: this.sqlType,
          sendEmail: this.sendEmail,
          displayRows: this.displayRows,
+          limit: this.limit,
          title: this.title,
          receivers: this.receivers.join(','),
          receiversCc: this.receiversCc.join(','),
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
@ -129,6 +129,9 @@ export default {
  'Send Email': 'Send Email',
  'Log display': 'Log display',
  'rows of result': 'rows of result',
+  'Max Numbers Return': 'Number of records to return',
+  'Max Numbers Return placeholder': 'Default is 10000, a large value may cause high pressure on the memory',
+  'Max Numbers Return required': 'Number of records to return parameter must be a number in the range of 0 - 2147483647',
  'Show Type': 'Show Type',
  Title: 'Title',
  'Please enter the title of email': 'Please enter the title of email',
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
@ -129,6 +129,9 @@ export default {
  'Send Email': '发送邮件',
  'Log display': '日志显示',
  'rows of result': '行查询结果',
+  'Max Numbers Return': '返回的记录行数',
+  'Max Numbers Return placeholder': '默认值10000,如果值过大可能会对内存造成较大压力',
+  'Max Numbers Return required': '返回的记录行数值必须是一个在0-2147483647范围内的整数',
  'Show Type': '呈现方式',
  Title: '主题',
  'Please enter the title of email': '请输入邮件主题',