From e6832220c35680aedae98d5ea88758b5d380bdfa Mon Sep 17 00:00:00 2001 From: zhuxt2015 <594754793@qq.com> Date: Mon, 26 Sep 2022 11:08:57 +0800 Subject: [PATCH] [Improvement][SQL Task]use default sql segment separator (#10869) --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/task/sql.md | 4 +- docs/docs/zh/guide/upgrade/incompatible.md | 2 +- .../dolphinscheduler/spi/enums/DbType.java | 8 +++ .../task/api/parameters/SqlParameters.java | 18 ------ .../plugin/task/sql/SqlSplitter.java | 64 ------------------- .../plugin/task/sql/SqlTask.java | 46 +++++++++++-- .../src/locales/en_US/project.ts | 2 - .../src/locales/zh_CN/project.ts | 2 - .../components/node/fields/use-sql-type.ts | 10 --- .../task/components/node/format-data.ts | 1 - .../task/components/node/tasks/use-sql.ts | 1 - .../projects/task/components/node/types.ts | 1 - 13 files changed, 51 insertions(+), 110 deletions(-) delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlSplitter.java diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index e17ae2f132..7b20d8fda4 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -9,4 +9,4 @@ This document records the incompatible updates between each version. You need to ## 3.0.0 * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) - +* Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) \ No newline at end of file diff --git a/docs/docs/zh/guide/task/sql.md b/docs/docs/zh/guide/task/sql.md index ba5cb817bc..fd55e2d5e6 100644 --- a/docs/docs/zh/guide/task/sql.md +++ b/docs/docs/zh/guide/task/sql.md @@ -20,9 +20,7 @@ SQL任务类型，用于连接数据库并执行相应SQL。 - sql类型：支持查询和非查询两种。 - 查询：支持 `DML select` 类型的命令，是有结果集返回的，可以指定邮件通知为表格、附件或表格附件三种模板； - 非查询：支持 `DDL`全部命令 和 `DML update、delete、insert` 三种类型的命令； - - 分段执行符号：提供在数据源不支持一次执行多段SQL语句时，拆分SQL语句的符号来进行多次调用数据源执行方法。 - 例子：1.当数据源选择Hive数据源时，不需要填写此参数。因为Hive数据源本身支持一次执行多段SQL语句； - 2.当数据源选择MySQL数据源时，并且要执行多段SQL语句时，需要填写此参数为分号 `;`。因为MySQL数据源不支持一次执行多段SQL语句； + - 默认采用`;\n`作为SQL分隔符,拆分成多段SQL语句执行。Hive支持一次执行多段SQL语句,故不会拆分。 - sql参数：输入参数格式为key1=value1;key2=value2… - sql语句：SQL语句 - UDF函数：对于HIVE类型的数据源，可以引用资源中心中创建的UDF函数，其他类型的数据源暂不支持UDF函数。 diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 59ec1e673e..6e97f7d316 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -9,4 +9,4 @@ ## 3.0.0 * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) - +* Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) \ No newline at end of file diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index c1e0144448..99c5ec8187 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -75,4 +75,12 @@ public enum DbType { public boolean isHive() { return this == DbType.HIVE; } + + /** + * support execute multiple segmented statements at a time + * @return + */ + public boolean isSupportMultipleStatement() { + return isHive() || this == DbType.SPARK; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java index 2bb7822a87..b3e2df055d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java @@ -114,15 +114,6 @@ public class SqlParameters extends AbstractParameters { private int limit; - /** - * segment separator - * - *

The segment separator is used - * when the data source does not support multi-segment SQL execution, - * and the client needs to split the SQL and execute it multiple times.

- */ - private String segmentSeparator; - public int getLimit() { return limit; } @@ -235,14 +226,6 @@ public class SqlParameters extends AbstractParameters { this.groupId = groupId; } - public String getSegmentSeparator() { - return segmentSeparator; - } - - public void setSegmentSeparator(String segmentSeparator) { - this.segmentSeparator = segmentSeparator; - } - @Override public boolean checkParameters() { return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(sql); @@ -310,7 +293,6 @@ public class SqlParameters extends AbstractParameters { + ", sendEmail=" + sendEmail + ", displayRows=" + displayRows + ", limit=" + limit - + ", segmentSeparator=" + segmentSeparator + ", udfs='" + udfs + '\'' + ", showType='" + showType + '\'' + ", connParams='" + connParams + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlSplitter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlSplitter.java deleted file mode 100644 index 0271b7c341..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlSplitter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.sql; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import com.google.common.base.Strings; - -public class SqlSplitter { - - private SqlSplitter() { - } - - private static final String LINE_SEPARATOR = "\n"; - - /** - * split sql by segment separator - *

The segment separator is used - * when the data source does not support multi-segment SQL execution, - * and the client needs to split the SQL and execute it multiple times.

- * @param sql - * @param segmentSeparator - * @return - */ - public static List split(String sql, String segmentSeparator) { - if (Strings.isNullOrEmpty(segmentSeparator)) { - return Collections.singletonList(sql); - } - - String[] lines = sql.split(LINE_SEPARATOR); - List segments = new ArrayList<>(); - StringBuilder stmt = new StringBuilder(); - for (String line : lines) { - if (line.trim().isEmpty() || line.startsWith("--")) { - continue; - } - stmt.append(LINE_SEPARATOR).append(line); - if (line.trim().endsWith(segmentSeparator)) { - segments.add(stmt.toString()); - stmt.setLength(0); - } - } - if (stmt.length() > 0) { - segments.add(stmt.toString()); - } - return segments; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index f58817efd1..2f766cca7f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.sql; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.collections4.CollectionUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; @@ -38,7 +35,8 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; + +import org.apache.commons.collections4.CollectionUtils; import java.sql.Connection; import java.sql.PreparedStatement; @@ -48,6 +46,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,6 +55,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.slf4j.Logger; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class SqlTask extends AbstractTask { /** @@ -88,6 +92,8 @@ public class SqlTask extends AbstractTask { public static final int TEST_FLAG_YES = 1; + private static final String SQL_SEPARATOR = ";\n"; + /** * Abstract Yarn Task * @@ -127,15 +133,18 @@ public class SqlTask extends AbstractTask { sqlParameters.getConnParams(), sqlParameters.getVarPool(), sqlParameters.getLimit()); + String separator = SQL_SEPARATOR; try { // get datasource baseConnectionParam = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( DbType.valueOf(sqlParameters.getType()), sqlTaskExecutionContext.getConnectionParams()); - + if (DbType.valueOf(sqlParameters.getType()).isSupportMultipleStatement()) { + separator = ""; + } // ready to execute SQL and parameter entity Map - List mainStatementSqlBinds = SqlSplitter.split(sqlParameters.getSql(), sqlParameters.getSegmentSeparator()) + List mainStatementSqlBinds = split(sqlParameters.getSql(), separator) .stream() .map(this::getSqlAndSqlParamsMap) .collect(Collectors.toList()); @@ -170,6 +179,31 @@ public class SqlTask extends AbstractTask { } + /** + * split sql by segment separator + *

The segment separator is used + * when the data source does not support multi-segment SQL execution, + * and the client needs to split the SQL and execute it multiple times.

+ * @param sql + * @param segmentSeparator + * @return + */ + public static List split(String sql, String segmentSeparator) { + if (StringUtils.isEmpty(segmentSeparator)) { + return Collections.singletonList(sql); + } + + String[] lines = sql.split(segmentSeparator); + List segments = new ArrayList<>(); + for (String line : lines) { + if (line.trim().isEmpty() || line.startsWith("--")) { + continue; + } + segments.add(line); + } + return segments; + } + /** * execute function and sql * diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 3697187a90..74218748a6 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -648,8 +648,6 @@ export default { emr_flow_define_json_tips: 'Please enter the definition of the job flow.', emr_steps_define_json: 'stepsDefineJson', emr_steps_define_json_tips: 'Please enter the definition of the emr step.', - segment_separator: 'Segment Execution Separator', - segment_separator_tips: 'Please enter the segment execution separator', zeppelin_note_id: 'zeppelinNoteId', zeppelin_note_id_tips: 'Please enter the note id of your zeppelin note', zeppelin_paragraph_id: 'zeppelinParagraphId', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 071a49e232..6ea2c23a96 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -642,8 +642,6 @@ export default { emr_flow_define_json_tips: '请输入工作流定义', emr_steps_define_json: 'stepsDefineJson', emr_steps_define_json_tips: '请输入EMR步骤定义', - segment_separator: '分段执行符号', - segment_separator_tips: '请输入分段执行符号', zeppelin_note_id: 'zeppelinNoteId', zeppelin_note_id_tips: '请输入zeppelin note id', zeppelin_paragraph_id: 'zeppelinParagraphId', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts index 0fcc5a3e62..1e93c9e208 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts @@ -24,7 +24,6 @@ import type { IJsonItem } from '../types' export function useSqlType(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() const querySpan = computed(() => (model.sqlType === '0' ? 6 : 0)) - const nonQuerySpan = computed(() => (model.sqlType === '1' ? 18 : 0)) const emailSpan = computed(() => model.sqlType === '0' && model.sendEmail ? 24 : 0 ) @@ -68,15 +67,6 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { required: true } }, - { - type: 'input', - field: 'segmentSeparator', - name: t('project.node.segment_separator'), - props: { - placeholder: t('project.node.segment_separator_tips') - }, - span: nonQuerySpan - }, { type: 'switch', field: 'sendEmail', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index edb6cd4043..5e3a3c7d99 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -189,7 +189,6 @@ export function formatParams(data: INodeData): { taskParams.sqlType = data.sqlType taskParams.preStatements = data.preStatements taskParams.postStatements = data.postStatements - taskParams.segmentSeparator = data.segmentSeparator taskParams.sendEmail = data.sendEmail taskParams.displayRows = data.displayRows if (data.sqlType === '0' && data.sendEmail) { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sql.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sql.ts index d477d340c9..8e235ac3d0 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sql.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sql.ts @@ -46,7 +46,6 @@ export function useSql({ timeout: 30, type: 'MYSQL', displayRows: 10, - segmentSeparator: '', sql: '', sqlType: '0', preStatements: [], diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index c013a7b2fb..e8732f3f34 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -253,7 +253,6 @@ interface ITaskParams { datasource?: string sql?: string sqlType?: string - segmentSeparator?: string sendEmail?: boolean displayRows?: number title?: string