diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java index db357e9d5c..98222a2c5a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java @@ -134,6 +134,7 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.other); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.other); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.other); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java index a613806460..81a4415f5d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java @@ -129,7 +129,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.clickhouse); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.clickhouse); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.clickhouse); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java index 1af61facd3..bc31bdcd49 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java @@ -135,7 +135,8 @@ public class DamengDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.dm); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.dm); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.dm); } private String transformOther(Map paramMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java index 2d67d91843..1d7c448355 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java @@ -129,7 +129,8 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.db2); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.db2); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.db2); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java index 36330c17c8..09d9f4b963 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java @@ -152,7 +152,8 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.hive); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.hive); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.hive); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java index 1da0cbadda..ef5e71729c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Assertions; @@ -94,4 +95,23 @@ public class HiveDataSourceProcessorTest { Assertions.assertEquals(DataSourceConstants.HIVE_VALIDATION_QUERY, hiveDatasourceProcessor.getValidationQuery()); } + + @Test + void splitAndRemoveComment() { + String sql = "create table if not exists test_ods.tb_test(\n" + + " `id` bigint COMMENT 'id', -- auto increment\n" + + " `user_name` string COMMENT 'username',\n" + + " `birthday` string COMMENT 'birthday',\n" + + " `gender` int COMMENT '1 male 2 female'\n" + + ") COMMENT 'user information table' PARTITIONED BY (`date_id` string);\n" + + "\n" + + "-- insert\n" + + "insert\n" + + " overwrite table test_ods.tb_test partition(date_id = '2024-03-28') -- partition\n" + + "values\n" + + " (1, 'Magic', '1990-10-01', '1');"; + List list = hiveDatasourceProcessor.splitAndRemoveComment(sql); + Assertions.assertEquals(list.size(), 2); + + } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java index b954defdd1..0c93b98212 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java @@ -177,7 +177,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.mysql); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.mysql); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.mysql); } private static boolean checkKeyIsLegitimate(String key) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java index b07b543c42..4d89f28eab 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java @@ -192,6 +192,7 @@ public class OceanBaseDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.oceanbase); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.oceanbase); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.oceanbase); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java index 2835d357ab..28a0aa2945 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java @@ -131,7 +131,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.postgresql); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.postgresql); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.postgresql); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java index 264f92c2b9..c3b73580dd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java @@ -128,7 +128,8 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.sqlserver); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.sqlserver); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.sqlserver); } private String transformOther(Map otherMap) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java index 77b10b5162..bd53acaaf8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java @@ -131,7 +131,8 @@ public class TrinoDataSourceProcessor extends AbstractDataSourceProcessor { @Override public List splitAndRemoveComment(String sql) { - return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.trino); + String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.trino); + return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.trino); } private String transformOther(Map otherMap) {