diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java index 37f46f94a4..c494496516 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.task.sql; import org.apache.dolphinscheduler.common.process.ResourceInfo; @@ -24,7 +25,7 @@ import java.util.ArrayList; import java.util.List; /** - * Sql/Hql parameter + * Sql/Hql parameter. */ public class SqlParameters extends AbstractParameters { /** @@ -103,6 +104,19 @@ public class SqlParameters extends AbstractParameters { */ private String receiversCc; + /** + * query result limit + */ + private int limit; + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + public String getType() { return type; } @@ -226,21 +240,22 @@ public class SqlParameters extends AbstractParameters { @Override public String toString() { - return "SqlParameters{" + - "type='" + type + '\'' + - ", datasource=" + datasource + - ", sql='" + sql + '\'' + - ", sqlType=" + sqlType + - ", sendEmail=" + sendEmail + - ", displayRows=" + displayRows + - ", udfs='" + udfs + '\'' + - ", showType='" + showType + '\'' + - ", connParams='" + connParams + '\'' + - ", title='" + title + '\'' + - ", receivers='" + receivers + '\'' + - ", receiversCc='" + receiversCc + '\'' + - ", preStatements=" + preStatements + - ", postStatements=" + postStatements + - '}'; + return "SqlParameters{" + + "type='" + type + '\'' + + ", datasource=" + datasource + + ", sql='" + sql + '\'' + + ", sqlType=" + sqlType + + ", sendEmail=" + sendEmail + + ", displayRows=" + displayRows + + ", limit=" + limit + + ", udfs='" + udfs + '\'' + + ", showType='" + showType + '\'' + + ", connParams='" + connParams + '\'' + + ", title='" + title + '\'' + + ", receivers='" + receivers + '\'' + + ", receiversCc='" + receiversCc + '\'' + + ", preStatements=" + preStatements + + ", postStatements=" + postStatements + + '}'; } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java index 3c1612c9b3..546bb633bb 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java @@ -32,6 +32,7 @@ public class SqlParametersTest { private final int sqlType = 0; private final Boolean sendEmail = true; private final int displayRows = 10; + private final int limit = 0; private final String showType = "TABLE"; private final String title = "sql test"; @@ -49,6 +50,7 @@ public class SqlParametersTest { sqlParameters.setDisplayRows(displayRows); sqlParameters.setShowType(showType); sqlParameters.setTitle(title); + sqlParameters.setLimit(limit); Assert.assertEquals(type, sqlParameters.getType()); Assert.assertEquals(sql, sqlParameters.getSql()); @@ -59,6 +61,7 @@ public class SqlParametersTest { Assert.assertEquals(displayRows, sqlParameters.getDisplayRows()); Assert.assertEquals(showType, sqlParameters.getShowType()); Assert.assertEquals(title, sqlParameters.getTitle()); + Assert.assertEquals(limit, sqlParameters.getLimit()); Assert.assertTrue(sqlParameters.checkParameters()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index d410560672..2b9b9074dc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,14 +16,9 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import static org.apache.dolphinscheduler.common.Constants.COMMA; -import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF; -import static org.apache.dolphinscheduler.common.Constants.PASSWORD; -import static org.apache.dolphinscheduler.common.Constants.SEMICOLON; -import static org.apache.dolphinscheduler.common.Constants.STATUS; -import static org.apache.dolphinscheduler.common.Constants.USER; -import static org.apache.dolphinscheduler.common.enums.DbType.HIVE; - +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -35,11 +30,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.sql.SqlBinds; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.EnumUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -50,30 +41,16 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.slf4j.Logger; -import org.apache.commons.lang.StringUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; +import java.sql.*; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.slf4j.Logger; - -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; +import static org.apache.dolphinscheduler.common.Constants.*; +import static org.apache.dolphinscheduler.common.enums.DbType.HIVE; /** * sql task */ @@ -97,11 +74,6 @@ public class SqlTask extends AbstractTask { */ private TaskExecutionContext taskExecutionContext; - /** - * default query sql limit - */ - private static final int LIMIT = 10000; - public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) { super(taskExecutionContext, logger); @@ -124,14 +96,15 @@ public class SqlTask extends AbstractTask { Thread.currentThread().setName(threadLoggerInfoName); logger.info("Full sql parameters: {}", sqlParameters); - logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}", + logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}, query max result limit : {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), sqlParameters.getLocalParams(), sqlParameters.getUdfs(), sqlParameters.getShowType(), - sqlParameters.getConnParams()); + sqlParameters.getConnParams(), + sqlParameters.getLimit()); try { SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext(); // load class @@ -282,7 +255,7 @@ public class SqlTask extends AbstractTask { int rowCount = 0; - while (rowCount < LIMIT && resultSet.next()) { + while (rowCount < sqlParameters.getLimit() && resultSet.next()) { JSONObject mapOfColValues = new JSONObject(true); for (int i = 1; i <= num; i++) { mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i)); diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index bfecfe3478..9a6cb574d4 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -46,6 +46,13 @@ + + *{{$t('Max Numbers Return')}} + + + + + {{$t('Show Type')}} @@ -187,6 +194,8 @@ sendEmail: false, // Display rows displayRows: 10, + // Max returned rows + limit: 10000, // Email title title: '', // Form/attachment @@ -209,6 +218,13 @@ createNodeId: Number }, methods: { + /** + * limit should't be empty;limit should be a non-negative number str; + * limit should be a number smaller or equal than Integer.MAX_VALUE in java. + */ + isLimitInvalid () { + return !this.limit || !/^(0|[1-9]\d*)$/.test(this.limit) || parseInt(this.limit, 10) > 2147483647 + }, /** * return sqlType */ @@ -270,6 +286,10 @@ this.$message.warning(`${i18n.$t('Mail subject required')}`) return false } + if (this.sqlType === '0' && this.isLimitInvalid()) { + this.$message.warning(`${i18n.$t('Max Numbers Return required')}`) + return false + } if (this.sqlType==0 && this.sendEmail && !this.receivers.length) { this.$message.warning(`${i18n.$t('Recipient required')}`) return false @@ -313,6 +333,7 @@ sqlType: this.sqlType, sendEmail: this.sendEmail, displayRows: this.displayRows, + limit: this.limit, title: this.title, receivers: this.receivers.join(','), receiversCc: this.receiversCc.join(','), @@ -390,6 +411,7 @@ sqlType: this.sqlType, sendEmail: this.sendEmail, displayRows: this.displayRows, + limit: this.limit, title: this.title, receivers: this.receivers.join(','), receiversCc: this.receiversCc.join(','), @@ -452,6 +474,7 @@ this.sqlType = o.params.sqlType this.sendEmail = o.params.sendEmail || false this.displayRows = o.params.displayRows || 10 + this.limit = o.params.limit || 10000 this.connParams = o.params.connParams || '' this.localParams = o.params.localParams || [] if(o.params.showType == '') { @@ -498,6 +521,7 @@ sqlType: this.sqlType, sendEmail: this.sendEmail, displayRows: this.displayRows, + limit: this.limit, title: this.title, receivers: this.receivers.join(','), receiversCc: this.receiversCc.join(','), diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index ec3125b0f9..4af6644af8 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -129,6 +129,9 @@ export default { 'Send Email': 'Send Email', 'Log display': 'Log display', 'rows of result': 'rows of result', + 'Max Numbers Return': 'Number of records to return', + 'Max Numbers Return placeholder': 'Default is 10000, a large value may cause high pressure on the memory', + 'Max Numbers Return required': 'Number of records to return parameter must be a number in the range of 0 - 2147483647', 'Show Type': 'Show Type', Title: 'Title', 'Please enter the title of email': 'Please enter the title of email', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index eeb6955802..90b1eca162 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -129,6 +129,9 @@ export default { 'Send Email': '发送邮件', 'Log display': '日志显示', 'rows of result': '行查询结果', + 'Max Numbers Return': '返回的记录行数', + 'Max Numbers Return placeholder': '默认值10000,如果值过大可能会对内存造成较大压力', + 'Max Numbers Return required': '返回的记录行数值必须是一个在0-2147483647范围内的整数', 'Show Type': '呈现方式', Title: '主题', 'Please enter the title of email': '请输入邮件主题',