Browse Source
* Time function analysis extension. * param add. * clear useless logs and update method notes * permission omission fix. * extending time functions to optimize static methods * e2e rerun.3.1.0-release
WangJPLeo
2 years ago
committed by
GitHub
19 changed files with 516 additions and 155 deletions
@ -0,0 +1,64 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
|
||||
import java.util.Date; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
public interface CuringGlobalParamsService { |
||||
|
||||
/** |
||||
* time function need expand |
||||
* @param placeholderName |
||||
* @return |
||||
*/ |
||||
boolean timeFunctionNeedExpand(String placeholderName); |
||||
|
||||
/** |
||||
* time function extension |
||||
* @param processInstanceId |
||||
* @param timezone |
||||
* @param placeholderName |
||||
* @return |
||||
*/ |
||||
String timeFunctionExtension(Integer processInstanceId, String timezone, String placeholderName); |
||||
|
||||
/** |
||||
* convert parameter placeholders |
||||
* @param val |
||||
* @param allParamMap |
||||
* @return |
||||
*/ |
||||
String convertParameterPlaceholders(String val, Map<String, String> allParamMap); |
||||
|
||||
/** |
||||
* curing global params |
||||
* @param processInstanceId |
||||
* @param globalParamMap |
||||
* @param globalParamList |
||||
* @param commandType |
||||
* @param scheduleTime |
||||
* @param timezone |
||||
* @return |
||||
*/ |
||||
String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap, List<Property> globalParamList, CommandType commandType, Date scheduleTime, String timezone); |
||||
} |
||||
@ -0,0 +1,96 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils; |
||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import java.util.Date; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Set; |
||||
|
||||
@Component |
||||
public class DolphinSchedulerCuringGlobalParams implements CuringGlobalParamsService { |
||||
|
||||
@Autowired |
||||
private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService; |
||||
|
||||
@Override |
||||
public String convertParameterPlaceholders(String val, Map<String, String> allParamMap) { |
||||
return ParameterUtils.convertParameterPlaceholders(val, allParamMap); |
||||
} |
||||
|
||||
@Override |
||||
public boolean timeFunctionNeedExpand(String placeholderName) { |
||||
return timePlaceholderResolverExpandService.timeFunctionNeedExpand(placeholderName); |
||||
} |
||||
|
||||
@Override |
||||
public String timeFunctionExtension(Integer processInstanceId, String timezone, String placeholderName) { |
||||
return timePlaceholderResolverExpandService.timeFunctionExtension(processInstanceId, timezone, placeholderName); |
||||
} |
||||
|
||||
@Override |
||||
public String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap, List<Property> globalParamList, CommandType commandType, Date scheduleTime, String timezone) { |
||||
if (globalParamList == null || globalParamList.isEmpty()) { |
||||
return null; |
||||
} |
||||
Map<String, String> globalMap = new HashMap<>(); |
||||
if (globalParamMap != null) { |
||||
globalMap.putAll(globalParamMap); |
||||
} |
||||
Map<String, String> allParamMap = new HashMap<>(); |
||||
//If it is a complement, a complement time needs to be passed in, according to the task type
|
||||
Map<String, String> timeParams = BusinessTimeUtils. |
||||
getBusinessTime(commandType, scheduleTime, timezone); |
||||
|
||||
if (timeParams != null) { |
||||
allParamMap.putAll(timeParams); |
||||
} |
||||
allParamMap.putAll(globalMap); |
||||
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); |
||||
Map<String, String> resolveMap = new HashMap<>(); |
||||
for (Map.Entry<String, String> entry : entries) { |
||||
String val = entry.getValue(); |
||||
if (val.startsWith("$")) { |
||||
String str = ""; |
||||
if (timeFunctionNeedExpand(val)) { |
||||
str = timeFunctionExtension(processInstanceId, timezone, val); |
||||
} else { |
||||
str = convertParameterPlaceholders(val, allParamMap); |
||||
} |
||||
resolveMap.put(entry.getKey(), str); |
||||
} |
||||
} |
||||
globalMap.putAll(resolveMap); |
||||
for (Property property : globalParamList) { |
||||
String val = globalMap.get(property.getProp()); |
||||
if (val != null) { |
||||
property.setValue(val); |
||||
} |
||||
} |
||||
return JSONUtils.toJsonString(globalParamList); |
||||
} |
||||
} |
||||
@ -0,0 +1,35 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
public interface TimePlaceholderResolverExpandService { |
||||
|
||||
/** |
||||
* check is need expand function |
||||
* @param placeholderName |
||||
* @return |
||||
*/ |
||||
boolean timeFunctionNeedExpand(String placeholderName); |
||||
|
||||
/** |
||||
* time function extension |
||||
* @param placeholderName |
||||
* @return |
||||
*/ |
||||
String timeFunctionExtension(Integer processInstanceId, String timeZone, String placeholderName); |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@Component |
||||
public class TimePlaceholderResolverExpandServiceImpl implements TimePlaceholderResolverExpandService { |
||||
|
||||
@Override |
||||
public boolean timeFunctionNeedExpand(String placeholderName) { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public String timeFunctionExtension(Integer processInstanceId, String timeZone, String placeholderName) { |
||||
return null; |
||||
} |
||||
} |
||||
@ -0,0 +1,151 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.Date; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
@RunWith(MockitoJUnitRunner.class) |
||||
public class CuringGlobalParamsServiceTest { |
||||
|
||||
private static final String placeHolderName = "$[yyyy-MM-dd-1]"; |
||||
|
||||
@Mock |
||||
private CuringGlobalParamsService curingGlobalParamsService; |
||||
|
||||
@InjectMocks |
||||
private DolphinSchedulerCuringGlobalParams dolphinSchedulerCuringGlobalParams; |
||||
|
||||
@Mock |
||||
private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService; |
||||
|
||||
@InjectMocks |
||||
private TimePlaceholderResolverExpandServiceImpl timePlaceholderResolverExpandServiceImpl; |
||||
|
||||
private final Map<String, String> globalParamMap = new HashMap<>(); |
||||
|
||||
@Before |
||||
public void init() { |
||||
globalParamMap.put("globalParams1", "Params1"); |
||||
} |
||||
|
||||
@Test |
||||
public void testConvertParameterPlaceholders() { |
||||
Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap)).thenReturn("2022-06-26"); |
||||
String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap); |
||||
Assert.assertNotNull(result); |
||||
} |
||||
|
||||
@Test |
||||
public void testTimeFunctionNeedExpand() { |
||||
boolean result = curingGlobalParamsService.timeFunctionNeedExpand(placeHolderName); |
||||
Assert.assertFalse(result); |
||||
} |
||||
|
||||
@Test |
||||
public void testTimeFunctionExtension() { |
||||
String result = curingGlobalParamsService.timeFunctionExtension(1, "", placeHolderName); |
||||
Assert.assertNull(result); |
||||
} |
||||
|
||||
@Test |
||||
public void testCuringGlobalParams() { |
||||
//define globalMap
|
||||
Map<String, String> globalParamMap = new HashMap<>(); |
||||
globalParamMap.put("globalParams1", "Params1"); |
||||
|
||||
//define globalParamList
|
||||
List<Property> globalParamList = new ArrayList<>(); |
||||
|
||||
//define scheduleTime
|
||||
Date scheduleTime = DateUtils.stringToDate("2019-12-20 00:00:00"); |
||||
|
||||
//test globalParamList is null
|
||||
String result = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); |
||||
Assert.assertNull(result); |
||||
Assert.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, null, CommandType.START_CURRENT_TASK_PROCESS, null, null)); |
||||
Assert.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, null, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null)); |
||||
|
||||
//test globalParamList is not null
|
||||
Property property = new Property("testGlobalParam", Direct.IN, DataType.VARCHAR, "testGlobalParam"); |
||||
globalParamList.add(property); |
||||
|
||||
String result2 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); |
||||
Assert.assertEquals(result2, JSONUtils.toJsonString(globalParamList)); |
||||
|
||||
String result3 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, null, null); |
||||
Assert.assertEquals(result3, JSONUtils.toJsonString(globalParamList)); |
||||
|
||||
String result4 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); |
||||
Assert.assertEquals(result4, JSONUtils.toJsonString(globalParamList)); |
||||
|
||||
//test var $ startsWith
|
||||
globalParamMap.put("bizDate", "${system.biz.date}"); |
||||
globalParamMap.put("b1zCurdate", "${system.biz.curdate}"); |
||||
|
||||
Property property2 = new Property("testParamList1", Direct.IN, DataType.VARCHAR, "testParamList"); |
||||
Property property3 = new Property("testParamList2", Direct.IN, DataType.VARCHAR, "{testParamList1}"); |
||||
Property property4 = new Property("testParamList3", Direct.IN, DataType.VARCHAR, "${b1zCurdate}"); |
||||
|
||||
globalParamList.add(property2); |
||||
globalParamList.add(property3); |
||||
globalParamList.add(property4); |
||||
|
||||
String result5 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); |
||||
Assert.assertEquals(result5, JSONUtils.toJsonString(globalParamList)); |
||||
|
||||
Property testStartParamProperty = new Property("testStartParam", Direct.IN, DataType.VARCHAR, ""); |
||||
globalParamList.add(testStartParamProperty); |
||||
Property testStartParam2Property = new Property("testStartParam2", Direct.IN, DataType.VARCHAR, "$[yyyy-MM-dd+1]"); |
||||
globalParamList.add(testStartParam2Property); |
||||
globalParamMap.put("testStartParam", ""); |
||||
globalParamMap.put("testStartParam2", "$[yyyy-MM-dd+1]"); |
||||
|
||||
Map<String, String> startParamMap = new HashMap<>(2); |
||||
startParamMap.put("testStartParam", "$[yyyyMMdd]"); |
||||
|
||||
for (Map.Entry<String, String> param : globalParamMap.entrySet()) { |
||||
String val = startParamMap.get(param.getKey()); |
||||
if (val != null) { |
||||
param.setValue(val); |
||||
} |
||||
} |
||||
|
||||
String result6 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); |
||||
Assert.assertTrue(result6.contains("20191220")); |
||||
} |
||||
} |
||||
@ -0,0 +1,51 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.expand; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
|
||||
@RunWith(MockitoJUnitRunner.class) |
||||
public class TimePlaceholderResolverExpandServiceTest { |
||||
|
||||
@Mock |
||||
private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService; |
||||
|
||||
@InjectMocks |
||||
private TimePlaceholderResolverExpandServiceImpl timePlaceholderResolverExpandServiceImpl; |
||||
|
||||
private static final String placeHolderName = "$[yyyy-MM-dd-1]"; |
||||
|
||||
@Test |
||||
public void testTimePlaceholderResolverExpandService() { |
||||
boolean checkResult = timePlaceholderResolverExpandService.timeFunctionNeedExpand(placeHolderName); |
||||
Assert.assertFalse(checkResult); |
||||
String resultString = timePlaceholderResolverExpandService.timeFunctionExtension(1, "", placeHolderName); |
||||
Assert.assertTrue(StringUtils.isEmpty(resultString)); |
||||
|
||||
boolean implCheckResult = timePlaceholderResolverExpandServiceImpl.timeFunctionNeedExpand(placeHolderName); |
||||
Assert.assertFalse(implCheckResult); |
||||
String implResultString = timePlaceholderResolverExpandServiceImpl.timeFunctionExtension(1, "", placeHolderName); |
||||
Assert.assertTrue(StringUtils.isEmpty(implResultString)); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue