|
|
|
@ -17,6 +17,8 @@
|
|
|
|
|
|
|
|
|
|
package org.apache.dolphinscheduler.plugin.task.sql; |
|
|
|
|
|
|
|
|
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
|
|
|
@ -24,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
|
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; |
|
|
|
|
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; |
|
|
|
|
import org.apache.dolphinscheduler.spi.enums.DbType; |
|
|
|
|
|
|
|
|
|
import java.util.HashMap; |
|
|
|
@ -33,6 +36,8 @@ import org.junit.jupiter.api.Assertions;
|
|
|
|
|
import org.junit.jupiter.api.BeforeEach; |
|
|
|
|
import org.junit.jupiter.api.Test; |
|
|
|
|
|
|
|
|
|
import com.google.common.collect.Lists; |
|
|
|
|
|
|
|
|
|
class SqlTaskTest { |
|
|
|
|
|
|
|
|
|
private SqlTask sqlTask; |
|
|
|
@ -57,21 +62,21 @@ class SqlTaskTest {
|
|
|
|
|
void testReplacingSqlWithoutParams() { |
|
|
|
|
String querySql = "select 1"; |
|
|
|
|
String expected = "select 1"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
|
void testReplacingSqlWithDollarSymbol() { |
|
|
|
|
String querySql = "select concat(amount, '$') as price from product"; |
|
|
|
|
String expected = "select concat(amount, '$') as price from product"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
|
void testReplacingHiveLoadSql() { |
|
|
|
|
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})"; |
|
|
|
|
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)"; |
|
|
|
|
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
Map<Integer, Property> sqlParamsMap = new HashMap<>(); |
|
|
|
|
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); |
|
|
|
@ -79,7 +84,7 @@ class SqlTaskTest {
|
|
|
|
|
expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
Map<String, Property> paramsMap = new HashMap<>(); |
|
|
|
|
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(hiveLoadSql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -87,38 +92,38 @@ class SqlTaskTest {
|
|
|
|
|
void testReplacingSelectSql() { |
|
|
|
|
String querySql = "select id from student where dt='${dt}'"; |
|
|
|
|
String expected = "select id from student where dt=?"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
Map<Integer, Property> sqlParamsMap = new HashMap<>(); |
|
|
|
|
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); |
|
|
|
|
expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
Map<String, Property> paramsMap = new HashMap<>(); |
|
|
|
|
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
|
|
|
|
|
querySql = "select id from student where dt=\"${dt}\""; |
|
|
|
|
expected = "select id from student where dt=?"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
sqlParamsMap.clear(); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
|
|
|
|
|
querySql = "select id from student where dt=${dt}"; |
|
|
|
|
expected = "select id from student where dt=?"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
sqlParamsMap.clear(); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
|
|
|
|
|
querySql = "select id from student where dt=${dt} and gender=1"; |
|
|
|
|
expected = "select id from student where dt=? and gender=1"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
sqlParamsMap.clear(); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -126,7 +131,7 @@ class SqlTaskTest {
|
|
|
|
|
void testReplacingSqlNonGreedy() { |
|
|
|
|
String querySql = "select id from student where year=${year} and month=${month} and gender=1"; |
|
|
|
|
String expected = "select id from student where year=? and month=? and gender=1"; |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); |
|
|
|
|
Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); |
|
|
|
|
|
|
|
|
|
Map<Integer, Property> sqlParamsMap = new HashMap<>(); |
|
|
|
|
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); |
|
|
|
@ -135,11 +140,37 @@ class SqlTaskTest {
|
|
|
|
|
Map<String, Property> paramsMap = new HashMap<>(); |
|
|
|
|
paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
|
void splitSql() { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
|
void testReplacingSqlHasQuestionMarkAndParams() { |
|
|
|
|
String querySql = |
|
|
|
|
"select id, concat('?', year) from student where year=${year} and month=${month} and gender in ('${gender}')"; |
|
|
|
|
String expected = |
|
|
|
|
"select id, concat('?', year) from student where year=? and month=? and gender in (?,?)"; |
|
|
|
|
|
|
|
|
|
Map<Integer, Property> sqlParamsMap = new HashMap<>(); |
|
|
|
|
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); |
|
|
|
|
expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12")); |
|
|
|
|
expectedSQLParamsMap.put(3, |
|
|
|
|
new Property("gender", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(1, 2)))); |
|
|
|
|
Map<String, Property> paramsMap = new HashMap<>(); |
|
|
|
|
paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); |
|
|
|
|
paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); |
|
|
|
|
paramsMap.put("gender", |
|
|
|
|
new Property("gender", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(1, 2)))); |
|
|
|
|
sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); |
|
|
|
|
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); |
|
|
|
|
|
|
|
|
|
String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, querySql); |
|
|
|
|
Assertions.assertEquals(4, sqlParamsMap.size()); |
|
|
|
|
Assertions.assertEquals(expected, formatSql); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|