Browse Source
* Global parameter and local parameter calculation external expansion. * k8s task ut fix. * TimePlaceholderUtils import DateUtils fix * follow the review comments to fix. * follow the review comments to fix. * e2e rerun3.1.0-release
WangJPLeo
2 years ago
committed by
GitHub
39 changed files with 344 additions and 424 deletions
@ -1,96 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.common.expand; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
|
||||||
import org.springframework.beans.factory.annotation.Autowired; |
|
||||||
import org.springframework.stereotype.Component; |
|
||||||
|
|
||||||
import java.util.Date; |
|
||||||
import java.util.HashMap; |
|
||||||
import java.util.List; |
|
||||||
import java.util.Map; |
|
||||||
import java.util.Set; |
|
||||||
|
|
||||||
@Component |
|
||||||
public class DolphinSchedulerCuringGlobalParams implements CuringGlobalParamsService { |
|
||||||
|
|
||||||
@Autowired |
|
||||||
private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService; |
|
||||||
|
|
||||||
@Override |
|
||||||
public String convertParameterPlaceholders(String val, Map<String, String> allParamMap) { |
|
||||||
return ParameterUtils.convertParameterPlaceholders(val, allParamMap); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public boolean timeFunctionNeedExpand(String placeholderName) { |
|
||||||
return timePlaceholderResolverExpandService.timeFunctionNeedExpand(placeholderName); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String timeFunctionExtension(Integer processInstanceId, String timezone, String placeholderName) { |
|
||||||
return timePlaceholderResolverExpandService.timeFunctionExtension(processInstanceId, timezone, placeholderName); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap, List<Property> globalParamList, CommandType commandType, Date scheduleTime, String timezone) { |
|
||||||
if (globalParamList == null || globalParamList.isEmpty()) { |
|
||||||
return null; |
|
||||||
} |
|
||||||
Map<String, String> globalMap = new HashMap<>(); |
|
||||||
if (globalParamMap != null) { |
|
||||||
globalMap.putAll(globalParamMap); |
|
||||||
} |
|
||||||
Map<String, String> allParamMap = new HashMap<>(); |
|
||||||
//If it is a complement, a complement time needs to be passed in, according to the task type
|
|
||||||
Map<String, String> timeParams = BusinessTimeUtils. |
|
||||||
getBusinessTime(commandType, scheduleTime, timezone); |
|
||||||
|
|
||||||
if (timeParams != null) { |
|
||||||
allParamMap.putAll(timeParams); |
|
||||||
} |
|
||||||
allParamMap.putAll(globalMap); |
|
||||||
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); |
|
||||||
Map<String, String> resolveMap = new HashMap<>(); |
|
||||||
for (Map.Entry<String, String> entry : entries) { |
|
||||||
String val = entry.getValue(); |
|
||||||
if (val.startsWith("$")) { |
|
||||||
String str = ""; |
|
||||||
if (timeFunctionNeedExpand(val)) { |
|
||||||
str = timeFunctionExtension(processInstanceId, timezone, val); |
|
||||||
} else { |
|
||||||
str = convertParameterPlaceholders(val, allParamMap); |
|
||||||
} |
|
||||||
resolveMap.put(entry.getKey(), str); |
|
||||||
} |
|
||||||
} |
|
||||||
globalMap.putAll(resolveMap); |
|
||||||
for (Property property : globalParamList) { |
|
||||||
String val = globalMap.get(property.getProp()); |
|
||||||
if (val != null) { |
|
||||||
property.setValue(val); |
|
||||||
} |
|
||||||
} |
|
||||||
return JSONUtils.toJsonString(globalParamList); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -0,0 +1,228 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.service.expand; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||||
|
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.ParameterUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; |
||||||
|
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Iterator; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.Set; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_TASK_EXECUTE_PATH; |
||||||
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_TASK_INSTANCE_ID; |
||||||
|
|
||||||
|
@Component |
||||||
|
public class CuringGlobalParams implements CuringParamsService { |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService; |
||||||
|
|
||||||
|
@Override |
||||||
|
public String convertParameterPlaceholders(String val, Map<String, String> allParamMap) { |
||||||
|
return ParameterUtils.convertParameterPlaceholders(val, allParamMap); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean timeFunctionNeedExpand(String placeholderName) { |
||||||
|
return timePlaceholderResolverExpandService.timeFunctionNeedExpand(placeholderName); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String timeFunctionExtension(Integer processInstanceId, String timezone, String placeholderName) { |
||||||
|
return timePlaceholderResolverExpandService.timeFunctionExtension(processInstanceId, timezone, placeholderName); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* here it is judged whether external expansion calculation is required and the calculation result is obtained |
||||||
|
* @param processInstanceId |
||||||
|
* @param globalParamMap |
||||||
|
* @param globalParamList |
||||||
|
* @param commandType |
||||||
|
* @param scheduleTime |
||||||
|
* @param timezone |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap, List<Property> globalParamList, CommandType commandType, Date scheduleTime, String timezone) { |
||||||
|
if (globalParamList == null || globalParamList.isEmpty()) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
Map<String, String> globalMap = new HashMap<>(); |
||||||
|
if (globalParamMap != null) { |
||||||
|
globalMap.putAll(globalParamMap); |
||||||
|
} |
||||||
|
Map<String, String> allParamMap = new HashMap<>(); |
||||||
|
//If it is a complement, a complement time needs to be passed in, according to the task type
|
||||||
|
Map<String, String> timeParams = BusinessTimeUtils. |
||||||
|
getBusinessTime(commandType, scheduleTime, timezone); |
||||||
|
|
||||||
|
if (timeParams != null) { |
||||||
|
allParamMap.putAll(timeParams); |
||||||
|
} |
||||||
|
allParamMap.putAll(globalMap); |
||||||
|
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); |
||||||
|
Map<String, String> resolveMap = new HashMap<>(); |
||||||
|
for (Map.Entry<String, String> entry : entries) { |
||||||
|
String val = entry.getValue(); |
||||||
|
if (val.startsWith(Constants.FUNCTION_START_WITH)) { |
||||||
|
String str = ""; |
||||||
|
// whether external scaling calculation is required
|
||||||
|
if (timeFunctionNeedExpand(val)) { |
||||||
|
str = timeFunctionExtension(processInstanceId, timezone, val); |
||||||
|
} else { |
||||||
|
str = convertParameterPlaceholders(val, allParamMap); |
||||||
|
} |
||||||
|
resolveMap.put(entry.getKey(), str); |
||||||
|
} |
||||||
|
} |
||||||
|
globalMap.putAll(resolveMap); |
||||||
|
for (Property property : globalParamList) { |
||||||
|
String val = globalMap.get(property.getProp()); |
||||||
|
if (val != null) { |
||||||
|
property.setValue(val); |
||||||
|
} |
||||||
|
} |
||||||
|
return JSONUtils.toJsonString(globalParamList); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* the global parameters and local parameters used in the worker will be prepared here. |
||||||
|
* |
||||||
|
* @param taskInstance |
||||||
|
* @param parameters |
||||||
|
* @param processInstance |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public Map<String, Property> paramParsingPreparation(@NonNull TaskInstance taskInstance, @NonNull AbstractParameters parameters, @NonNull ProcessInstance processInstance) { |
||||||
|
// assign value to definedParams here
|
||||||
|
Map<String,String> globalParamsMap = setGlobalParamsMap(processInstance); |
||||||
|
Map<String, Property> globalParams = ParamUtils.getUserDefParamsMap(globalParamsMap); |
||||||
|
CommandType commandType = processInstance.getCmdTypeIfComplement(); |
||||||
|
Date scheduleTime = processInstance.getScheduleTime(); |
||||||
|
|
||||||
|
// combining local and global parameters
|
||||||
|
Map<String, Property> localParams = parameters.getInputLocalParametersMap(); |
||||||
|
|
||||||
|
//stream pass params
|
||||||
|
Map<String, Property> varParams = parameters.getVarPoolMap(); |
||||||
|
|
||||||
|
if (globalParams.isEmpty() && localParams.isEmpty() && varParams.isEmpty()) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
// if it is a complement,
|
||||||
|
// you need to pass in the task instance id to locate the time
|
||||||
|
// of the process instance complement
|
||||||
|
Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam()); |
||||||
|
String timeZone = cmdParam.get(Constants.SCHEDULE_TIMEZONE); |
||||||
|
Map<String,String> params = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timeZone); |
||||||
|
|
||||||
|
if (globalParamsMap != null) { |
||||||
|
params.putAll(globalParamsMap); |
||||||
|
} |
||||||
|
|
||||||
|
if (StringUtils.isNotBlank(taskInstance.getExecutePath())) { |
||||||
|
params.put(PARAMETER_TASK_EXECUTE_PATH, taskInstance.getExecutePath()); |
||||||
|
} |
||||||
|
params.put(PARAMETER_TASK_INSTANCE_ID, Integer.toString(taskInstance.getId())); |
||||||
|
|
||||||
|
if (varParams.size() != 0) { |
||||||
|
globalParams.putAll(varParams); |
||||||
|
} |
||||||
|
if (localParams.size() != 0) { |
||||||
|
globalParams.putAll(localParams); |
||||||
|
} |
||||||
|
|
||||||
|
Iterator<Map.Entry<String, Property>> iter = globalParams.entrySet().iterator(); |
||||||
|
while (iter.hasNext()) { |
||||||
|
Map.Entry<String, Property> en = iter.next(); |
||||||
|
Property property = en.getValue(); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(property.getValue()) && property.getValue().startsWith(Constants.FUNCTION_START_WITH)) { |
||||||
|
/** |
||||||
|
* local parameter refers to global parameter with the same name |
||||||
|
* note: the global parameters of the process instance here are solidified parameters, |
||||||
|
* and there are no variables in them. |
||||||
|
*/ |
||||||
|
String val = property.getValue(); |
||||||
|
// whether external scaling calculation is required
|
||||||
|
if (timeFunctionNeedExpand(val)) { |
||||||
|
val = timeFunctionExtension(taskInstance.getProcessInstanceId(), timeZone, val); |
||||||
|
} else { |
||||||
|
val = convertParameterPlaceholders(val, params); |
||||||
|
} |
||||||
|
property.setValue(val); |
||||||
|
} |
||||||
|
} |
||||||
|
if (MapUtils.isEmpty(globalParams)) { |
||||||
|
globalParams = new HashMap<>(); |
||||||
|
} |
||||||
|
// put schedule time param to params map
|
||||||
|
Map<String, Property> paramsMap = preBuildBusinessParams(processInstance); |
||||||
|
if (MapUtils.isNotEmpty(paramsMap)) { |
||||||
|
globalParams.putAll(paramsMap); |
||||||
|
} |
||||||
|
return globalParams; |
||||||
|
} |
||||||
|
|
||||||
|
private Map<String, String> setGlobalParamsMap(ProcessInstance processInstance) { |
||||||
|
Map<String, String> globalParamsMap = new HashMap<>(16); |
||||||
|
|
||||||
|
// global params string
|
||||||
|
String globalParamsStr = processInstance.getGlobalParams(); |
||||||
|
if (globalParamsStr != null) { |
||||||
|
List<Property> globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); |
||||||
|
globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); |
||||||
|
} |
||||||
|
return globalParamsMap; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, Property> preBuildBusinessParams(ProcessInstance processInstance) { |
||||||
|
Map<String, Property> paramsMap = new HashMap<>(); |
||||||
|
// replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
|
||||||
|
if (processInstance.getScheduleTime() != null) { |
||||||
|
Date date = processInstance.getScheduleTime(); |
||||||
|
String dateTime = DateUtils.format(date, Constants.PARAMETER_FORMAT_TIME, null); |
||||||
|
Property p = new Property(); |
||||||
|
p.setValue(dateTime); |
||||||
|
p.setProp(Constants.PARAMETER_DATETIME); |
||||||
|
paramsMap.put(Constants.PARAMETER_DATETIME, p); |
||||||
|
} |
||||||
|
return paramsMap; |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue