Wenjun Ruan
6 months ago
committed by
GitHub
60 changed files with 495 additions and 480 deletions
@ -0,0 +1,107 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.task.api.parser; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
import org.apache.commons.lang3.tuple.ImmutablePair; |
||||
import org.apache.commons.lang3.tuple.Pair; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
/** |
||||
* Used to parse ${setValue()} and #{setValue()} from given lines. |
||||
*/ |
||||
@Slf4j |
||||
@NotThreadSafe |
||||
public class TaskOutputParameterParser { |
||||
|
||||
private final Map<String, String> taskOutputParams = new HashMap<>(); |
||||
|
||||
private List<String> currentTaskOutputParam; |
||||
|
||||
public void appendParseLog(String log) { |
||||
if (log == null) { |
||||
return; |
||||
} |
||||
|
||||
if (currentTaskOutputParam != null) { |
||||
// continue to parse the rest of line
|
||||
int i = log.indexOf(")}"); |
||||
if (i == -1) { |
||||
// the end of var pool not found
|
||||
currentTaskOutputParam.add(log); |
||||
} else { |
||||
// the end of var pool found
|
||||
currentTaskOutputParam.add(log.substring(0, i + 2)); |
||||
Pair<String, String> keyValue = parseOutputParam(String.join("\n", currentTaskOutputParam)); |
||||
if (keyValue.getKey() != null && keyValue.getValue() != null) { |
||||
taskOutputParams.put(keyValue.getKey(), keyValue.getValue()); |
||||
} |
||||
currentTaskOutputParam = null; |
||||
// continue to parse the rest of line
|
||||
if (i + 2 != log.length()) { |
||||
appendParseLog(log.substring(i + 2)); |
||||
} |
||||
} |
||||
return; |
||||
} |
||||
|
||||
int indexOfVarPoolBegin = log.indexOf("${setValue("); |
||||
if (indexOfVarPoolBegin == -1) { |
||||
indexOfVarPoolBegin = log.indexOf("#{setValue("); |
||||
} |
||||
if (indexOfVarPoolBegin == -1) { |
||||
return; |
||||
} |
||||
currentTaskOutputParam = new ArrayList<>(); |
||||
appendParseLog(log.substring(indexOfVarPoolBegin)); |
||||
} |
||||
|
||||
public Map<String, String> getTaskOutputParams() { |
||||
return taskOutputParams; |
||||
} |
||||
|
||||
// #{setValue(xx=xx)}
|
||||
protected Pair<String, String> parseOutputParam(String outputParam) { |
||||
if (StringUtils.isEmpty(outputParam)) { |
||||
log.info("The task output param is empty"); |
||||
return ImmutablePair.nullPair(); |
||||
} |
||||
if ((!outputParam.startsWith("${setValue(") && !outputParam.startsWith("#{setValue(")) |
||||
|| !outputParam.endsWith(")}")) { |
||||
log.info("The task output param {} should start with '${setValue(' or '#{setValue(' and end with ')}'", |
||||
outputParam); |
||||
return ImmutablePair.nullPair(); |
||||
} |
||||
String keyValueExpression = outputParam.substring(11, outputParam.length() - 2); |
||||
if (!keyValueExpression.contains("=")) { |
||||
log.warn("The task output param {} should composite with key=value", outputParam); |
||||
return ImmutablePair.nullPair(); |
||||
} |
||||
|
||||
String[] keyValue = keyValueExpression.split("=", 2); |
||||
return ImmutablePair.of(keyValue[0], keyValue[1]); |
||||
} |
||||
|
||||
} |
||||
@ -1,47 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
|
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
|
||||
import lombok.experimental.UtilityClass; |
||||
|
||||
@UtilityClass |
||||
public class VarPoolUtils { |
||||
|
||||
static final Pattern DSVALUE_REGEX = Pattern.compile(TaskConstants.DSVALUE_REGEX); |
||||
public static final String VAR_SUFFIX = ")dsVal}"; |
||||
|
||||
public static final String VAR_DELIMITER = "$VarPool$"; |
||||
/** |
||||
* find var pool |
||||
* |
||||
* @param line |
||||
* @return |
||||
*/ |
||||
public static String findVarPool(String line) { |
||||
Matcher matcher = DSVALUE_REGEX.matcher(line); |
||||
if (matcher.find()) { |
||||
return matcher.group(1); |
||||
} |
||||
return null; |
||||
} |
||||
} |
||||
@ -0,0 +1,77 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.parser; |
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||
|
||||
import java.io.IOException; |
||||
import java.net.URI; |
||||
import java.net.URISyntaxException; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Paths; |
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import com.google.common.collect.ImmutableMap; |
||||
|
||||
class TaskOutputParameterParserTest { |
||||
|
||||
@Test |
||||
void testEmptyLog() throws IOException, URISyntaxException { |
||||
List<String> varPools = getLogs("/outputParam/emptyVarPoolLog.txt"); |
||||
TaskOutputParameterParser taskOutputParameterParser = new TaskOutputParameterParser(); |
||||
varPools.forEach(taskOutputParameterParser::appendParseLog); |
||||
Assertions.assertTrue(taskOutputParameterParser.getTaskOutputParams().isEmpty()); |
||||
} |
||||
|
||||
@Test |
||||
void testOneLineLog() throws IOException, URISyntaxException { |
||||
List<String> varPools = getLogs("/outputParam/onelineVarPoolLog.txt"); |
||||
TaskOutputParameterParser taskOutputParameterParser = new TaskOutputParameterParser(); |
||||
varPools.forEach(taskOutputParameterParser::appendParseLog); |
||||
assertEquals(ImmutableMap.of("name", "name=tom"), taskOutputParameterParser.getTaskOutputParams()); |
||||
} |
||||
|
||||
@Test |
||||
void testOneVarPollInMultiLineLog() throws IOException, URISyntaxException { |
||||
List<String> varPools = getLogs("/outputParam/oneVarPollInMultiLineLog.txt"); |
||||
TaskOutputParameterParser taskOutputParameterParser = new TaskOutputParameterParser(); |
||||
varPools.forEach(taskOutputParameterParser::appendParseLog); |
||||
assertEquals(ImmutableMap.of("sql", |
||||
"select * from table\n" + |
||||
"where\n" + |
||||
"id = 1\n"), |
||||
taskOutputParameterParser.getTaskOutputParams()); |
||||
} |
||||
|
||||
@Test |
||||
void testVarPollInMultiLineLog() throws IOException, URISyntaxException { |
||||
List<String> varPools = getLogs("/outputParam/multipleVarPoll.txt"); |
||||
TaskOutputParameterParser taskOutputParameterParser = new TaskOutputParameterParser(); |
||||
varPools.forEach(taskOutputParameterParser::appendParseLog); |
||||
assertEquals(ImmutableMap.of("name", "tom", "age", "1"), taskOutputParameterParser.getTaskOutputParams()); |
||||
} |
||||
|
||||
private List<String> getLogs(String file) throws IOException, URISyntaxException { |
||||
URI uri = TaskOutputParameterParserTest.class.getResource(file).toURI(); |
||||
return Files.lines(Paths.get(uri)).collect(Collectors.toList()); |
||||
} |
||||
} |
||||
@ -1,50 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import java.util.HashMap; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
class VarPoolUtilsTest { |
||||
|
||||
@Test |
||||
void findVar() { |
||||
HashMap<String, String> tcs = new HashMap<>(); |
||||
tcs.put("${(set_val=123)dsVal}", "set_val=123"); |
||||
tcs.put("1970-01-01 ${(set_val=123)dsVal}", "set_val=123"); |
||||
tcs.put("1970-01-01 ${(set_val=123)dsVal}123", null); |
||||
tcs.put("${(set_val=123}dsVal", null); |
||||
tcs.put("#{(set_val=123)dsVal}", "set_val=123"); |
||||
tcs.put("1970-01-01 #{(set_val=123)dsVal}", "set_val=123"); |
||||
tcs.put("1970-01-01 #{(set_val=123)dsVal}123", null); |
||||
tcs.put("#{(set_val=123)dsVal}123", null); |
||||
tcs.put("#{(set_val=123dsVal}", null); |
||||
|
||||
tcs.put("${(set_val=123)dsVal}${(set_val=456)dsVal}", "set_val=123)dsVal}${(set_val=456"); |
||||
tcs.put("1970-01-01$#{(set_val=123)dsVal}", "set_val=123"); |
||||
tcs.put("1970-01-01{(set_val=123)dsVal}123", null); |
||||
tcs.put("1970-01-01$#{(${(set_val=123)})dsVal}", "${(set_val=123)}"); |
||||
tcs.put("1970-01-01$#{(${(set_val=123\\)})dsVal}", "${(set_val=123\\)}"); |
||||
|
||||
for (String tc : tcs.keySet()) { |
||||
Assertions.assertEquals(tcs.get(tc), VarPoolUtils.findVarPool(tc)); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,16 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
@ -0,0 +1,19 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
INFO: ${setValue(name=tom)} |
||||
INFO: ${setValue(age=1)} |
||||
@ -0,0 +1,21 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
${setValue(sql=select * from table |
||||
where |
||||
id = 1 |
||||
)} |
||||
@ -0,0 +1,18 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
${setValue(name=name=tom)} |
||||
Loading…
Reference in new issue