diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java index 37f46f94a4..c494496516 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.task.sql; import org.apache.dolphinscheduler.common.process.ResourceInfo; @@ -24,7 +25,7 @@ import java.util.ArrayList; import java.util.List; /** - * Sql/Hql parameter + * Sql/Hql parameter. */ public class SqlParameters extends AbstractParameters { /** @@ -103,6 +104,19 @@ public class SqlParameters extends AbstractParameters { */ private String receiversCc; + /** + * query result limit + */ + private int limit; + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + public String getType() { return type; } @@ -226,21 +240,22 @@ public class SqlParameters extends AbstractParameters { @Override public String toString() { - return "SqlParameters{" + - "type='" + type + '\'' + - ", datasource=" + datasource + - ", sql='" + sql + '\'' + - ", sqlType=" + sqlType + - ", sendEmail=" + sendEmail + - ", displayRows=" + displayRows + - ", udfs='" + udfs + '\'' + - ", showType='" + showType + '\'' + - ", connParams='" + connParams + '\'' + - ", title='" + title + '\'' + - ", receivers='" + receivers + '\'' + - ", receiversCc='" + receiversCc + '\'' + - ", preStatements=" + preStatements + - ", postStatements=" + postStatements + - '}'; + return "SqlParameters{" + + "type='" + type + '\'' + + ", datasource=" + datasource + + ", sql='" + sql + '\'' + + ", sqlType=" + sqlType + + ", sendEmail=" + sendEmail + + ", displayRows=" + displayRows + + ", limit=" + limit + + ", udfs='" + udfs + '\'' + + ", showType='" + showType + '\'' + + ", connParams='" + connParams + '\'' + + ", title='" + title + '\'' + + ", receivers='" + receivers + '\'' + + ", receiversCc='" + receiversCc + '\'' + + ", preStatements=" + preStatements + + ", postStatements=" + postStatements + + '}'; } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java index 3c1612c9b3..546bb633bb 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java @@ -32,6 +32,7 @@ public class SqlParametersTest { private final int sqlType = 0; private final Boolean sendEmail = true; private final int displayRows = 10; + private final int limit = 0; private final String showType = "TABLE"; private final String title = "sql test"; @@ -49,6 +50,7 @@ public class SqlParametersTest { sqlParameters.setDisplayRows(displayRows); sqlParameters.setShowType(showType); sqlParameters.setTitle(title); + sqlParameters.setLimit(limit); Assert.assertEquals(type, sqlParameters.getType()); Assert.assertEquals(sql, sqlParameters.getSql()); @@ -59,6 +61,7 @@ public class SqlParametersTest { Assert.assertEquals(displayRows, sqlParameters.getDisplayRows()); Assert.assertEquals(showType, sqlParameters.getShowType()); Assert.assertEquals(title, sqlParameters.getTitle()); + Assert.assertEquals(limit, sqlParameters.getLimit()); Assert.assertTrue(sqlParameters.checkParameters()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index d410560672..2b9b9074dc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,14 +16,9 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import static org.apache.dolphinscheduler.common.Constants.COMMA; -import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF; -import static org.apache.dolphinscheduler.common.Constants.PASSWORD; -import static org.apache.dolphinscheduler.common.Constants.SEMICOLON; -import static org.apache.dolphinscheduler.common.Constants.STATUS; -import static org.apache.dolphinscheduler.common.Constants.USER; -import static org.apache.dolphinscheduler.common.enums.DbType.HIVE; - +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -35,11 +30,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.sql.SqlBinds; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.EnumUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -50,30 +41,16 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.slf4j.Logger; -import org.apache.commons.lang.StringUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; +import java.sql.*; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.slf4j.Logger; - -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; +import static org.apache.dolphinscheduler.common.Constants.*; +import static org.apache.dolphinscheduler.common.enums.DbType.HIVE; /** * sql task */ @@ -97,11 +74,6 @@ public class SqlTask extends AbstractTask { */ private TaskExecutionContext taskExecutionContext; - /** - * default query sql limit - */ - private static final int LIMIT = 10000; - public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) { super(taskExecutionContext, logger); @@ -124,14 +96,15 @@ public class SqlTask extends AbstractTask { Thread.currentThread().setName(threadLoggerInfoName); logger.info("Full sql parameters: {}", sqlParameters); - logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}", + logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}, query max result limit : {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), sqlParameters.getLocalParams(), sqlParameters.getUdfs(), sqlParameters.getShowType(), - sqlParameters.getConnParams()); + sqlParameters.getConnParams(), + sqlParameters.getLimit()); try { SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext(); // load class @@ -282,7 +255,7 @@ public class SqlTask extends AbstractTask { int rowCount = 0; - while (rowCount < LIMIT && resultSet.next()) { + while (rowCount < sqlParameters.getLimit() && resultSet.next()) { JSONObject mapOfColValues = new JSONObject(true); for (int i = 1; i <= num; i++) { mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i)); diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index bfecfe3478..9a6cb574d4 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -46,6 +46,13 @@ + +
*{{$t('Max Numbers Return')}}
+
+ + +
+