Browse Source

[Fix-13381][plugin] fix error while parsing hive load sql (#13378)

* [fix][task plugin][task sql] fix parsing error while using variables sometimes parsing exception happened if task sql contains a variable followed by \" or \' .

for example:
input sql: load inpath '/tmp/test_table/dt=${dt}'
into table test_table partition(dt=${dt})

after replace: preparing : load inpath '/tmp/test_table/dt=?
into table test_table partition(dt=?)
3.2.0-release
Alex Ting 1 year ago committed by GitHub
parent
commit
606b56403e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      docs/docs/en/guide/upgrade/incompatible.md
  2. 1
      docs/docs/zh/guide/upgrade/incompatible.md
  3. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
  4. 96
      dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java

1
docs/docs/en/guide/upgrade/incompatible.md

@ -4,6 +4,7 @@ This document records the incompatible updates between each version. You need to
## dev
* Change regex matching sql params in SQL task plugin ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
* Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).

1
docs/docs/zh/guide/upgrade/incompatible.md

@ -4,6 +4,7 @@
## dev
* 更新了SQL任务中用于匹配变量的正则表达式 ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
* Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).

2
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java

@ -38,7 +38,7 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
public String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
/**
* varPool string

96
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.sql;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class SqlTaskTest {
private SqlTask sqlTask;
@BeforeEach
void setup() {
DataSourceParameters parameters = new DataSourceParameters();
parameters.setType(DbType.HIVE);
parameters.setResourceType(ResourceType.DATASOURCE.name());
ResourceParametersHelper resourceParametersHelper = new ResourceParametersHelper();
resourceParametersHelper.put(ResourceType.DATASOURCE, 1, parameters);
TaskExecutionContext ctx = new TaskExecutionContext();
ctx.setResourceParametersHelper(resourceParametersHelper);
ctx.setTaskParams("{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"}");
sqlTask = new SqlTask(ctx);
}
@Test
void testReplacingSqlWithoutParams() {
String querySql = "select 1";
String expected = "select 1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
}
@Test
void testReplacingSqlWithDollarSymbol() {
String querySql = "select concat(amount, '$') as price from product";
String expected = "select concat(amount, '$') as price from product";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
}
@Test
void testReplacingHiveLoadSql() {
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
}
@Test
void testReplacingSelectSql() {
String querySql = "select id from student where dt='${dt}'";
String expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
querySql = "select id from student where dt=\"${dt}\"";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
querySql = "select id from student where dt=${dt}";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
querySql = "select id from student where dt=${dt} and gender=1";
expected = "select id from student where dt=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
}
@Test
void testReplacingSqlNonGreedy() {
String querySql = "select id from student where year=${year} and month=${month} and gender=1";
String expected = "select id from student where year=? and month=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
}
}
Loading…
Cancel
Save