Browse Source

[Bug][Worker] Global and local parameters are not replaced in SQL components (#14926)

Signed-off-by: Gallardot <gallardot@apache.org>
3.2.1-prepare
Gallardot 1 year ago committed by GitHub
parent
commit
f6a6794289
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 10
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
  2. 45
      dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java

10
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java

@ -38,7 +38,9 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class); protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}"; private static String groupName1 = "paramName1";
private static String groupName2 = "paramName2";
public String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2);
/** /**
* varPool string * varPool string
@ -198,7 +200,11 @@ public abstract class AbstractTask {
int index = 1; int index = 1;
while (m.find()) { while (m.find()) {
String paramName = m.group(1); String paramName = m.group(groupName1);
if (paramName == null) {
paramName = m.group(groupName2);
}
Property prop = paramsPropsMap.get(paramName); Property prop = paramsPropsMap.get(paramName);
if (prop == null) { if (prop == null) {

45
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java

@ -18,11 +18,17 @@
package org.apache.dolphinscheduler.plugin.task.sql; package org.apache.dolphinscheduler.plugin.task.sql;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -66,6 +72,15 @@ class SqlTaskTest {
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})"; String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)"; String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
} }
@Test @Test
@ -74,17 +89,37 @@ class SqlTaskTest {
String expected = "select id from student where dt=?"; String expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=\"${dt}\""; querySql = "select id from student where dt=\"${dt}\"";
expected = "select id from student where dt=?"; expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=${dt}"; querySql = "select id from student where dt=${dt}";
expected = "select id from student where dt=?"; expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=${dt} and gender=1"; querySql = "select id from student where dt=${dt} and gender=1";
expected = "select id from student where dt=? and gender=1"; expected = "select id from student where dt=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
} }
@Test @Test
@ -92,5 +127,15 @@ class SqlTaskTest {
String querySql = "select id from student where year=${year} and month=${month} and gender=1"; String querySql = "select id from student where year=${year} and month=${month} and gender=1";
String expected = "select id from student where year=? and month=? and gender=1"; String expected = "select id from student where year=? and month=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12"));
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
} }
} }

Loading…
Cancel
Save