dailidong
4 years ago
114 changed files with 3549 additions and 1706 deletions
@ -0,0 +1,47 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.api.utils; |
||||||
|
|
||||||
|
import java.util.regex.Matcher; |
||||||
|
import java.util.regex.Pattern; |
||||||
|
|
||||||
|
/** |
||||||
|
* This is Regex expression utils. |
||||||
|
*/ |
||||||
|
public class RegexUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* check number regex expression |
||||||
|
*/ |
||||||
|
private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$"; |
||||||
|
|
||||||
|
private RegexUtils() { |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* check if the input is number |
||||||
|
* |
||||||
|
* @param str input |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static boolean isNumeric(String str) { |
||||||
|
Pattern pattern = Pattern.compile(CHECK_NUMBER); |
||||||
|
Matcher isNum = pattern.matcher(str); |
||||||
|
return isNum.matches(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,39 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.api.utils; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
/** |
||||||
|
* RegexUtils test case |
||||||
|
*/ |
||||||
|
public class RegexUtilsTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testIsNumeric() { |
||||||
|
String num1 = "123467854678"; |
||||||
|
boolean numeric = RegexUtils.isNumeric(num1); |
||||||
|
Assert.assertTrue(numeric); |
||||||
|
|
||||||
|
String num2 = "0.0.01"; |
||||||
|
boolean numeric2 = RegexUtils.isNumeric(num2); |
||||||
|
Assert.assertFalse(numeric2); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,78 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.common.task; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
public class TaskParams { |
||||||
|
|
||||||
|
private String rawScript; |
||||||
|
private Map<String, String>[] localParams; |
||||||
|
|
||||||
|
public void setRawScript(String rawScript) { |
||||||
|
this.rawScript = rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLocalParams(Map<String, String>[] localParams) { |
||||||
|
this.localParams = localParams; |
||||||
|
} |
||||||
|
|
||||||
|
public String getRawScript() { |
||||||
|
return rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLocalParamValue(String prop, Object value) { |
||||||
|
if (localParams == null || value == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
for (int i = 0; i < localParams.length; i++) { |
||||||
|
if (localParams[i].get("prop").equals(prop)) { |
||||||
|
localParams[i].put("value", (String)value); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void setLocalParamValue(Map<String, Object> propToValue) { |
||||||
|
if (localParams == null || propToValue == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
for (int i = 0; i < localParams.length; i++) { |
||||||
|
String prop = localParams[i].get("prop"); |
||||||
|
if (propToValue.containsKey(prop)) { |
||||||
|
localParams[i].put("value",(String)propToValue.get(prop)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public String getLocalParamValue(String prop) { |
||||||
|
if (localParams == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
for (int i = 0; i < localParams.length; i++) { |
||||||
|
String tmpProp = localParams[i].get("prop"); |
||||||
|
if (tmpProp.equals(prop)) { |
||||||
|
return localParams[i].get("value"); |
||||||
|
} |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
public Map<String, String>[] getLocalParams() { |
||||||
|
return localParams; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,124 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.model.TaskNode; |
||||||
|
import org.apache.dolphinscheduler.common.task.TaskParams; |
||||||
|
|
||||||
|
import java.text.ParseException; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
public class VarPoolUtils { |
||||||
|
/** |
||||||
|
* getTaskNodeLocalParam |
||||||
|
* @param taskNode taskNode |
||||||
|
* @param prop prop |
||||||
|
* @return localParamForProp |
||||||
|
*/ |
||||||
|
public static Object getTaskNodeLocalParam(TaskNode taskNode, String prop) { |
||||||
|
String taskParamsJson = taskNode.getParams(); |
||||||
|
TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); |
||||||
|
if (taskParams == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
return taskParams.getLocalParamValue(prop); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setTaskNodeLocalParams |
||||||
|
* @param taskNode taskNode |
||||||
|
* @param prop LocalParamName |
||||||
|
* @param value LocalParamValue |
||||||
|
*/ |
||||||
|
public static void setTaskNodeLocalParams(TaskNode taskNode, String prop, Object value) { |
||||||
|
String taskParamsJson = taskNode.getParams(); |
||||||
|
TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); |
||||||
|
if (taskParams == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
taskParams.setLocalParamValue(prop, value); |
||||||
|
taskNode.setParams(JSONUtils.toJsonString(taskParams)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setTaskNodeLocalParams |
||||||
|
* @param taskNode taskNode |
||||||
|
* @param propToValue propToValue |
||||||
|
*/ |
||||||
|
public static void setTaskNodeLocalParams(TaskNode taskNode, Map<String, Object> propToValue) { |
||||||
|
String taskParamsJson = taskNode.getParams(); |
||||||
|
TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); |
||||||
|
if (taskParams == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
taskParams.setLocalParamValue(propToValue); |
||||||
|
taskNode.setParams(JSONUtils.toJsonString(taskParams)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* convertVarPoolToMap |
||||||
|
* @param propToValue propToValue |
||||||
|
* @param varPool varPool |
||||||
|
* @throws ParseException ParseException |
||||||
|
*/ |
||||||
|
public static void convertVarPoolToMap(Map<String, Object> propToValue, String varPool) throws ParseException { |
||||||
|
if (varPool == null || propToValue == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
String[] splits = varPool.split("\\$VarPool\\$"); |
||||||
|
for (String kv : splits) { |
||||||
|
String[] kvs = kv.split(","); |
||||||
|
if (kvs.length == 2) { |
||||||
|
propToValue.put(kvs[0], kvs[1]); |
||||||
|
} else { |
||||||
|
throw new ParseException(kv, 2); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* convertPythonScriptPlaceholders |
||||||
|
* @param rawScript rawScript |
||||||
|
* @return String |
||||||
|
* @throws StringIndexOutOfBoundsException StringIndexOutOfBoundsException |
||||||
|
*/ |
||||||
|
public static String convertPythonScriptPlaceholders(String rawScript) throws StringIndexOutOfBoundsException { |
||||||
|
int len = "${setShareVar(${".length(); |
||||||
|
int scriptStart = 0; |
||||||
|
while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { |
||||||
|
int start = -1; |
||||||
|
int end = rawScript.indexOf('}', scriptStart + len); |
||||||
|
String prop = rawScript.substring(scriptStart + len, end); |
||||||
|
|
||||||
|
start = rawScript.indexOf(',', end); |
||||||
|
end = rawScript.indexOf(')', start); |
||||||
|
|
||||||
|
String value = rawScript.substring(start + 1, end); |
||||||
|
|
||||||
|
start = rawScript.indexOf('}', start) + 1; |
||||||
|
end = rawScript.length(); |
||||||
|
|
||||||
|
String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); |
||||||
|
|
||||||
|
rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); |
||||||
|
|
||||||
|
scriptStart += replaceScript.length(); |
||||||
|
} |
||||||
|
return rawScript; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,73 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.model.TaskNode; |
||||||
|
|
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
public class VarPoolUtilsTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(VarPoolUtilsTest.class); |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSetTaskNodeLocalParams() { |
||||||
|
String taskJson = "{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\"," |
||||||
|
+ "\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-75298\",\"maxRetryTimes\":0,\"name\":\"a1\"," |
||||||
|
+ "\"params\":\"{\\\"rawScript\\\":\\\"print(\\\\\\\"this is python task \\\\\\\",${p0})\\\"," |
||||||
|
+ "\\\"localParams\\\":[{\\\"prop\\\":\\\"p1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"1\\\"}]," |
||||||
|
+ "\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\"," |
||||||
|
+ "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\"," |
||||||
|
+ "\"type\":\"PYTHON\",\"workerGroup\":\"default\"}"; |
||||||
|
TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); |
||||||
|
|
||||||
|
VarPoolUtils.setTaskNodeLocalParams(taskNode, "p1", "test1"); |
||||||
|
Assert.assertEquals(VarPoolUtils.getTaskNodeLocalParam(taskNode, "p1"), "test1"); |
||||||
|
|
||||||
|
ConcurrentHashMap<String, Object> propToValue = new ConcurrentHashMap<String, Object>(); |
||||||
|
propToValue.put("p1", "test2"); |
||||||
|
|
||||||
|
VarPoolUtils.setTaskNodeLocalParams(taskNode, propToValue); |
||||||
|
Assert.assertEquals(VarPoolUtils.getTaskNodeLocalParam(taskNode, "p1"), "test2"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testConvertVarPoolToMap() throws Exception { |
||||||
|
String varPool = "p1,66$VarPool$p2,69$VarPool$"; |
||||||
|
ConcurrentHashMap<String, Object> propToValue = new ConcurrentHashMap<String, Object>(); |
||||||
|
VarPoolUtils.convertVarPoolToMap(propToValue, varPool); |
||||||
|
Assert.assertEquals((String)propToValue.get("p1"), "66"); |
||||||
|
Assert.assertEquals((String)propToValue.get("p2"), "69"); |
||||||
|
logger.info(propToValue.toString()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testConvertPythonScriptPlaceholders() throws Exception { |
||||||
|
String rawScript = "print(${p1});\n${setShareVar(${p1},3)};\n${setShareVar(${p2},4)};"; |
||||||
|
rawScript = VarPoolUtils.convertPythonScriptPlaceholders(rawScript); |
||||||
|
Assert.assertEquals(rawScript, "print(${p1});\n" |
||||||
|
+ "print(\"${{setValue({},{})}}\".format(\"p1\",3));\n" |
||||||
|
+ "print(\"${{setValue({},{})}}\".format(\"p2\",4));"); |
||||||
|
logger.info(rawScript); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,19 @@ |
|||||||
|
Copyright (c) 2012-2014 Chris Pettitt |
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||||
|
of this software and associated documentation files (the "Software"), to deal |
||||||
|
in the Software without restriction, including without limitation the rights |
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||||
|
copies of the Software, and to permit persons to whom the Software is |
||||||
|
furnished to do so, subject to the following conditions: |
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in |
||||||
|
all copies or substantial portions of the Software. |
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||||
|
THE SOFTWARE. |
||||||
@ -1 +1,105 @@ |
|||||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
public enum CommandType {
/**
* remove task log request,
*/
REMOVE_TAK_LOG_REQUEST,
/**
* remove task log response
*/
REMOVE_TAK_LOG_RESPONSE,
/**
* roll view log request
*/
ROLL_VIEW_LOG_REQUEST,
/**
* roll view log response
*/
ROLL_VIEW_LOG_RESPONSE,
/**
* view whole log request
*/
VIEW_WHOLE_LOG_REQUEST,
/**
* view whole log response
*/
VIEW_WHOLE_LOG_RESPONSE,
/**
* get log bytes request
*/
GET_LOG_BYTES_REQUEST,
/**
* get log bytes response
*/
GET_LOG_BYTES_RESPONSE,
WORKER_REQUEST,
MASTER_RESPONSE,
/**
* execute task request
*/
TASK_EXECUTE_REQUEST,
/**
* execute task ack
*/
TASK_EXECUTE_ACK,
/**
* execute task response
*/
TASK_EXECUTE_RESPONSE,
/**
* kill task
*/
TASK_KILL_REQUEST,
/**
* kill task response
*/
TASK_KILL_RESPONSE,
/**
* ping
*/
PING,
/**
* pong
*/
PONG;
} |
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.remote.command; |
||||||
|
|
||||||
|
public enum CommandType { |
||||||
|
|
||||||
|
/** |
||||||
|
* remove task log request, |
||||||
|
*/ |
||||||
|
REMOVE_TAK_LOG_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* remove task log response |
||||||
|
*/ |
||||||
|
REMOVE_TAK_LOG_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* roll view log request |
||||||
|
*/ |
||||||
|
ROLL_VIEW_LOG_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* roll view log response |
||||||
|
*/ |
||||||
|
ROLL_VIEW_LOG_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* view whole log request |
||||||
|
*/ |
||||||
|
VIEW_WHOLE_LOG_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* view whole log response |
||||||
|
*/ |
||||||
|
VIEW_WHOLE_LOG_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* get log bytes request |
||||||
|
*/ |
||||||
|
GET_LOG_BYTES_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* get log bytes response |
||||||
|
*/ |
||||||
|
GET_LOG_BYTES_RESPONSE, |
||||||
|
|
||||||
|
|
||||||
|
WORKER_REQUEST, |
||||||
|
MASTER_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* execute task request |
||||||
|
*/ |
||||||
|
TASK_EXECUTE_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* execute task ack |
||||||
|
*/ |
||||||
|
TASK_EXECUTE_ACK, |
||||||
|
|
||||||
|
/** |
||||||
|
* execute task response |
||||||
|
*/ |
||||||
|
TASK_EXECUTE_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* kill task |
||||||
|
*/ |
||||||
|
TASK_KILL_REQUEST, |
||||||
|
|
||||||
|
/** |
||||||
|
* kill task response |
||||||
|
*/ |
||||||
|
TASK_KILL_RESPONSE, |
||||||
|
|
||||||
|
/** |
||||||
|
* HEART_BEAT |
||||||
|
*/ |
||||||
|
HEART_BEAT, |
||||||
|
|
||||||
|
/** |
||||||
|
* ping |
||||||
|
*/ |
||||||
|
PING, |
||||||
|
|
||||||
|
/** |
||||||
|
* pong |
||||||
|
*/ |
||||||
|
PONG; |
||||||
|
} |
||||||
|
|||||||
@ -0,0 +1,43 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.remote.utils.Host; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
/** |
||||||
|
* host test |
||||||
|
*/ |
||||||
|
public class HostTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHostWarmUp() { |
||||||
|
Host host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000))); |
||||||
|
Assert.assertEquals(50, host.getWeight()); |
||||||
|
host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 10 * 1000))); |
||||||
|
Assert.assertEquals(100, host.getWeight()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHost() { |
||||||
|
Host host = Host.of("192.158.2.2:22"); |
||||||
|
Assert.assertEquals(22, host.getPort()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,114 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.task; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.powermock.api.mockito.PowerMockito; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({SpringApplicationContext.class}) |
||||||
|
public class TaskManagerTest { |
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(TaskManagerTest.class); |
||||||
|
|
||||||
|
private TaskExecutionContext taskExecutionContext; |
||||||
|
|
||||||
|
private Logger taskLogger; |
||||||
|
|
||||||
|
private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
// init task execution context, logger
|
||||||
|
taskExecutionContext = new TaskExecutionContext(); |
||||||
|
taskExecutionContext.setProcessId(12345); |
||||||
|
taskExecutionContext.setProcessDefineId(1); |
||||||
|
taskExecutionContext.setProcessInstanceId(1); |
||||||
|
taskExecutionContext.setTaskInstanceId(1); |
||||||
|
taskExecutionContext.setTaskType(""); |
||||||
|
taskExecutionContext.setFirstSubmitTime(new Date()); |
||||||
|
taskExecutionContext.setDelayTime(0); |
||||||
|
taskExecutionContext.setLogPath("/tmp/test.log"); |
||||||
|
taskExecutionContext.setHost("localhost"); |
||||||
|
taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); |
||||||
|
|
||||||
|
taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( |
||||||
|
LoggerUtils.TASK_LOGGER_INFO_PREFIX, |
||||||
|
taskExecutionContext.getProcessDefineId(), |
||||||
|
taskExecutionContext.getProcessInstanceId(), |
||||||
|
taskExecutionContext.getTaskInstanceId() |
||||||
|
)); |
||||||
|
|
||||||
|
taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); |
||||||
|
taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); |
||||||
|
|
||||||
|
PowerMockito.mockStatic(SpringApplicationContext.class); |
||||||
|
PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) |
||||||
|
.thenReturn(taskExecutionContextCacheManager); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testNewTask() { |
||||||
|
|
||||||
|
taskExecutionContext.setTaskType("SHELL"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("WATERDROP"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("HTTP"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("MR"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("SPARK"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("FLINK"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("PYTHON"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("DATAX"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
taskExecutionContext.setTaskType("SQOOP"); |
||||||
|
Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Test(expected = IllegalArgumentException.class) |
||||||
|
public void testNewTaskIsNull() { |
||||||
|
taskExecutionContext.setTaskType(null); |
||||||
|
TaskManager.newTask(taskExecutionContext,taskLogger); |
||||||
|
} |
||||||
|
|
||||||
|
@Test(expected = IllegalArgumentException.class) |
||||||
|
public void testNewTaskIsNotExists() { |
||||||
|
taskExecutionContext.setTaskType("XXX"); |
||||||
|
TaskManager.newTask(taskExecutionContext,taskLogger); |
||||||
|
} |
||||||
|
} |
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,116 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.service.process; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||||
|
import org.apache.dolphinscheduler.common.enums.WarningType; |
||||||
|
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.Command; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||||
|
|
||||||
|
/** |
||||||
|
* process service test |
||||||
|
*/ |
||||||
|
public class ProcessServiceTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testCreateSubCommand() { |
||||||
|
ProcessService processService = new ProcessService(); |
||||||
|
ProcessInstance parentInstance = new ProcessInstance(); |
||||||
|
parentInstance.setProcessDefinitionId(1); |
||||||
|
parentInstance.setWarningType(WarningType.SUCCESS); |
||||||
|
parentInstance.setWarningGroupId(0); |
||||||
|
|
||||||
|
TaskInstance task = new TaskInstance(); |
||||||
|
task.setTaskJson("{\"params\":{\"processDefinitionId\":100}}"); |
||||||
|
task.setId(10); |
||||||
|
|
||||||
|
ProcessInstance childInstance = null; |
||||||
|
ProcessInstanceMap instanceMap = new ProcessInstanceMap(); |
||||||
|
instanceMap.setParentProcessInstanceId(1); |
||||||
|
instanceMap.setParentTaskInstanceId(10); |
||||||
|
Command command = null; |
||||||
|
|
||||||
|
//father history: start; child null == command type: start
|
||||||
|
parentInstance.setHistoryCmd("START_PROCESS"); |
||||||
|
parentInstance.setCommandType(CommandType.START_PROCESS); |
||||||
|
command = processService.createSubProcessCommand( |
||||||
|
parentInstance, childInstance, instanceMap, task |
||||||
|
); |
||||||
|
Assert.assertEquals(CommandType.START_PROCESS, command.getCommandType()); |
||||||
|
|
||||||
|
//father history: start,start failure; child null == command type: start
|
||||||
|
parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); |
||||||
|
parentInstance.setHistoryCmd("START_PROCESS,START_FAILURE_TASK_PROCESS"); |
||||||
|
command = processService.createSubProcessCommand( |
||||||
|
parentInstance, childInstance, instanceMap, task |
||||||
|
); |
||||||
|
Assert.assertEquals(CommandType.START_PROCESS, command.getCommandType()); |
||||||
|
|
||||||
|
//father history: scheduler,start failure; child null == command type: scheduler
|
||||||
|
parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); |
||||||
|
parentInstance.setHistoryCmd("SCHEDULER,START_FAILURE_TASK_PROCESS"); |
||||||
|
command = processService.createSubProcessCommand( |
||||||
|
parentInstance, childInstance, instanceMap, task |
||||||
|
); |
||||||
|
Assert.assertEquals(CommandType.SCHEDULER, command.getCommandType()); |
||||||
|
|
||||||
|
//father history: complement,start failure; child null == command type: complement
|
||||||
|
|
||||||
|
String startString = "2020-01-01 00:00:00"; |
||||||
|
String endString = "2020-01-10 00:00:00"; |
||||||
|
parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); |
||||||
|
parentInstance.setHistoryCmd("COMPLEMENT_DATA,START_FAILURE_TASK_PROCESS"); |
||||||
|
Map<String,String> complementMap = new HashMap<>(); |
||||||
|
complementMap.put(Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE, startString); |
||||||
|
complementMap.put(Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE, endString); |
||||||
|
parentInstance.setCommandParam(JSONUtils.toJsonString(complementMap)); |
||||||
|
command = processService.createSubProcessCommand( |
||||||
|
parentInstance, childInstance, instanceMap, task |
||||||
|
); |
||||||
|
Assert.assertEquals(CommandType.COMPLEMENT_DATA, command.getCommandType()); |
||||||
|
|
||||||
|
JsonNode complementDate = JSONUtils.parseObject(command.getCommandParam()); |
||||||
|
Date start = DateUtils.stringToDate(complementDate.get(Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE).asText()); |
||||||
|
Date end = DateUtils.stringToDate(complementDate.get(Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE).asText()); |
||||||
|
Assert.assertEquals(startString, DateUtils.dateToString(start)); |
||||||
|
Assert.assertEquals(endString, DateUtils.dateToString(end)); |
||||||
|
|
||||||
|
//father history: start,failure,start failure; child not null == command type: start failure
|
||||||
|
childInstance = new ProcessInstance(); |
||||||
|
parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); |
||||||
|
parentInstance.setHistoryCmd("START_PROCESS,START_FAILURE_TASK_PROCESS"); |
||||||
|
command = processService.createSubProcessCommand( |
||||||
|
parentInstance, childInstance, instanceMap, task |
||||||
|
); |
||||||
|
Assert.assertEquals(CommandType.START_FAILURE_TASK_PROCESS, command.getCommandType()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,218 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
<template> |
||||||
|
<div class="timeout-alarm-model"> |
||||||
|
<div class="clearfix list"> |
||||||
|
<div class="text-box"> |
||||||
|
<span>{{$t('Timeout alarm')}}</span> |
||||||
|
</div> |
||||||
|
<div class="cont-box"> |
||||||
|
<label class="label-box"> |
||||||
|
<div style="padding-top: 5px;"> |
||||||
|
<x-switch v-model="enable" @on-click="_onSwitch(0, $event)" :disabled="isDetails"></x-switch> |
||||||
|
</div> |
||||||
|
</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div class="clearfix list" v-if="enable"> |
||||||
|
<div class="text-box"> |
||||||
|
<span>{{$t('Waiting Dependent start')}}</span> |
||||||
|
</div> |
||||||
|
<div class="cont-box"> |
||||||
|
<label class="label-box"> |
||||||
|
<div style="padding: 5px 0;"> |
||||||
|
<x-switch v-model="waitStartTimeout.enable" @on-click="_onSwitch(1, $event)" :disabled="isDetails"></x-switch> |
||||||
|
</div> |
||||||
|
</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div class="clearfix list" v-if="enable && waitStartTimeout.enable"> |
||||||
|
<div class="cont-box"> |
||||||
|
<label class="label-box"> |
||||||
|
<span class="text-box"> |
||||||
|
<span>{{$t('Timeout period')}}</span> |
||||||
|
</span> |
||||||
|
<x-input v-model="waitStartTimeout.interval" style="width: 100px;" :disabled="isDetails" maxlength="9"> |
||||||
|
<span slot="append">{{$t('Minute')}}</span> |
||||||
|
</x-input> |
||||||
|
<span class="text-box"> |
||||||
|
<span>{{$t('Check interval')}}</span> |
||||||
|
</span> |
||||||
|
<x-input v-model="waitStartTimeout.checkInterval" style="width: 100px;" :disabled="isDetails" maxlength="9"> |
||||||
|
<span slot="append">{{$t('Minute')}}</span> |
||||||
|
</x-input> |
||||||
|
<span class="text-box"> |
||||||
|
<span>{{$t('Timeout strategy')}}</span> |
||||||
|
</span> |
||||||
|
<div style="padding-top: 6px;"> |
||||||
|
<x-checkbox-group v-model="waitStartTimeout.strategy"> |
||||||
|
<x-checkbox label="FAILED" :disabled="true">{{$t('Timeout failure')}}</x-checkbox> |
||||||
|
</x-checkbox-group> |
||||||
|
</div> |
||||||
|
</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div class="clearfix list" v-if="enable"> |
||||||
|
<div class="text-box"> |
||||||
|
<span>{{$t('Waiting Dependent complete')}}</span> |
||||||
|
</div> |
||||||
|
<div class="cont-box"> |
||||||
|
<label class="label-box"> |
||||||
|
<div style="padding: 5px 0;"> |
||||||
|
<x-switch v-model="waitCompleteTimeout.enable" @on-click="_onSwitch(2, $event)" :disabled="isDetails"></x-switch> |
||||||
|
</div> |
||||||
|
</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div class="clearfix list" v-if="enable && waitCompleteTimeout.enable"> |
||||||
|
<div class="cont-box"> |
||||||
|
<label class="label-box"> |
||||||
|
<span class="text-box"> |
||||||
|
<span>{{$t('Timeout period')}}</span> |
||||||
|
</span> |
||||||
|
<x-input v-model="waitCompleteTimeout.interval" style="width: 100px;" :disabled="isDetails" maxlength="9"> |
||||||
|
<span slot="append">{{$t('Minute')}}</span> |
||||||
|
</x-input> |
||||||
|
<span class="text-box"> |
||||||
|
<span>{{$t('Timeout strategy')}}</span> |
||||||
|
</span> |
||||||
|
<div style="padding-top: 6px;"> |
||||||
|
<x-checkbox-group v-model="waitCompleteTimeout.strategy"> |
||||||
|
<x-checkbox label="WARN" :disabled="isDetails">{{$t('Timeout alarm')}}</x-checkbox> |
||||||
|
<x-checkbox label="FAILED" :disabled="isDetails">{{$t('Timeout failure')}}</x-checkbox> |
||||||
|
</x-checkbox-group> |
||||||
|
</div> |
||||||
|
</label> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</template> |
||||||
|
<script> |
||||||
|
import _ from 'lodash' |
||||||
|
import disabledState from '@/module/mixin/disabledState' |
||||||
|
|
||||||
|
export default { |
||||||
|
name: 'form-dependent-timeout', |
||||||
|
data () { |
||||||
|
return { |
||||||
|
// Timeout display hiding |
||||||
|
enable: false, |
||||||
|
waitStartTimeout: { |
||||||
|
enable: false, |
||||||
|
// Timeout strategy |
||||||
|
strategy: ['FAILED'], |
||||||
|
// Timeout period |
||||||
|
interval: null, |
||||||
|
checkInterval: null |
||||||
|
}, |
||||||
|
waitCompleteTimeout: { |
||||||
|
enable: false, |
||||||
|
// Timeout strategy |
||||||
|
strategy: [], |
||||||
|
// Timeout period |
||||||
|
interval: null |
||||||
|
} |
||||||
|
} |
||||||
|
}, |
||||||
|
mixins: [disabledState], |
||||||
|
props: { |
||||||
|
backfillItem: Object |
||||||
|
}, |
||||||
|
methods: { |
||||||
|
_onSwitch (p, is) { |
||||||
|
// reset timeout setting when switch timeout on/off. |
||||||
|
// p = 0 for timeout switch; p = 1 for wait start timeout switch; p = 2 for wait complete timeout switch. |
||||||
|
if (p === 1 || p === 0) { |
||||||
|
this.waitStartTimeout.interval = is ? 30 : null |
||||||
|
this.waitStartTimeout.checkInterval = is ? 1 : null |
||||||
|
} |
||||||
|
if (p === 2 || p === 0) { |
||||||
|
this.waitCompleteTimeout.strategy = is ? ['WARN'] : [] |
||||||
|
this.waitCompleteTimeout.interval = is ? 30 : null |
||||||
|
} |
||||||
|
}, |
||||||
|
_verification () { |
||||||
|
// Verification timeout policy |
||||||
|
if (this.enable |
||||||
|
&& (this.waitCompleteTimeout.enable && !this.waitCompleteTimeout.strategy.length) |
||||||
|
|| (this.waitStartTimeout.enable && !this.waitStartTimeout.strategy.length)) { |
||||||
|
this.$message.warning(`${this.$t('Timeout strategy must be selected')}`) |
||||||
|
return false |
||||||
|
} |
||||||
|
// Verify timeout duration Non 0 positive integer |
||||||
|
const reg = /^[1-9]\d*$/ |
||||||
|
if (this.enable |
||||||
|
&& (this.waitCompleteTimeout.enable && !reg.test(this.waitCompleteTimeout.interval)) |
||||||
|
|| (this.waitStartTimeout.enable && (!reg.test(this.waitStartTimeout.interval || !reg.test(this.waitStartTimeout.checkInterval))))) { |
||||||
|
this.$message.warning(`${this.$t('Timeout must be a positive integer')}`) |
||||||
|
return false |
||||||
|
} |
||||||
|
// Verify timeout duration longer than check interval |
||||||
|
if (this.enable && this.waitStartTimeout.enable && this.waitStartTimeout.checkInterval >= this.waitStartTimeout.interval) { |
||||||
|
this.$message.warning(`${this.$t('Timeout must be longer than check interval')}`) |
||||||
|
return false |
||||||
|
} |
||||||
|
this.$emit('on-timeout', { |
||||||
|
waitStartTimeout: { |
||||||
|
strategy: 'FAILED', |
||||||
|
interval: parseInt(this.waitStartTimeout.interval), |
||||||
|
checkInterval: parseInt(this.waitStartTimeout.checkInterval), |
||||||
|
enable: this.waitStartTimeout.enable |
||||||
|
}, |
||||||
|
waitCompleteTimeout: { |
||||||
|
strategy: (() => { |
||||||
|
// Handling checkout sequence |
||||||
|
let strategy = this.waitCompleteTimeout.strategy |
||||||
|
if (strategy.length === 2 && strategy[0] === 'FAILED') { |
||||||
|
return [strategy[1], strategy[0]].join(',') |
||||||
|
} else { |
||||||
|
return strategy.join(',') |
||||||
|
} |
||||||
|
})(), |
||||||
|
interval: parseInt(this.waitCompleteTimeout.interval), |
||||||
|
enable: this.waitCompleteTimeout.enable |
||||||
|
} |
||||||
|
}) |
||||||
|
return true |
||||||
|
} |
||||||
|
}, |
||||||
|
watch: { |
||||||
|
}, |
||||||
|
created () { |
||||||
|
let o = this.backfillItem |
||||||
|
// Non-null objects represent backfill |
||||||
|
if (!_.isEmpty(o)) { |
||||||
|
if (o.timeout) { |
||||||
|
this.enable = true |
||||||
|
this.waitCompleteTimeout.enable = o.timeout.enable || false |
||||||
|
this.waitCompleteTimeout.strategy = _.split(o.timeout.strategy, ',') || ['WARN'] |
||||||
|
this.waitCompleteTimeout.interval = o.timeout.interval || null |
||||||
|
} |
||||||
|
if (o.waitStartTimeout) { |
||||||
|
this.enable = true |
||||||
|
this.waitStartTimeout.enable = o.waitStartTimeout.enable || false |
||||||
|
this.waitStartTimeout.strategy = ['FAILED'] |
||||||
|
this.waitStartTimeout.interval = o.waitStartTimeout.interval || null |
||||||
|
this.waitStartTimeout.checkInterval = o.waitStartTimeout.checkInterval || null |
||||||
|
} |
||||||
|
} |
||||||
|
}, |
||||||
|
mounted () { |
||||||
|
}, |
||||||
|
components: {} |
||||||
|
} |
||||||
|
</script> |
||||||
@ -0,0 +1,108 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
<template> |
||||||
|
<div class="pre_tasks-model"> |
||||||
|
<div class="clearfix list"> |
||||||
|
<div class="text-box"> |
||||||
|
<span>{{$t('Pre tasks')}}</span> |
||||||
|
</div> |
||||||
|
<div class="cont-box"> |
||||||
|
<div class="label-box"> |
||||||
|
<x-select |
||||||
|
ref="preTasksSelector" |
||||||
|
style="width: 100%;" |
||||||
|
filterable |
||||||
|
multiple |
||||||
|
v-model="preTasks" |
||||||
|
:disabled="isDetails" |
||||||
|
:id="preTasksSelectorId"> |
||||||
|
<x-option |
||||||
|
v-for="task in preTaskList" |
||||||
|
:key="task.id" |
||||||
|
:value="task.id" |
||||||
|
:label="task.name"> |
||||||
|
</x-option> |
||||||
|
</x-select> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</template> |
||||||
|
<script> |
||||||
|
import disabledState from '@/module/mixin/disabledState' |
||||||
|
export default { |
||||||
|
name: 'pre_tasks', |
||||||
|
mixins: [disabledState], |
||||||
|
props: { |
||||||
|
backfillItem: Object |
||||||
|
}, |
||||||
|
data () { |
||||||
|
return { |
||||||
|
preTasksSelectorId: '_preTasksSelectorId', // Refresh target vue-component by changing id |
||||||
|
preTasks: [], |
||||||
|
preTasksOld: [], |
||||||
|
} |
||||||
|
}, |
||||||
|
mounted () { |
||||||
|
this.preTasks = this.backfillItem['preTasks'] || this.preTasks |
||||||
|
this.preTasksOld = this.preTasks |
||||||
|
|
||||||
|
// Refresh target vue-component by changing id |
||||||
|
this.$nextTick(() => { |
||||||
|
this.preTasksSelectorId = 'preTasksSelectorId' |
||||||
|
}) |
||||||
|
}, |
||||||
|
computed: { |
||||||
|
preTaskList: function () { |
||||||
|
let currentTaskId = this.backfillItem['id'] || this.id |
||||||
|
let cacheTasks = Object.assign({}, this.store.state.dag.tasks) |
||||||
|
let keys = Object.keys(cacheTasks) |
||||||
|
for (let i = 0; i < keys.length; i++) { |
||||||
|
let key = keys[i] |
||||||
|
if ((!cacheTasks[key].id || !cacheTasks[key].name) || (currentTaskId && cacheTasks[key].id === currentTaskId)) { |
||||||
|
// Clean undefined and current task data |
||||||
|
delete cacheTasks[key] |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return cacheTasks |
||||||
|
}, |
||||||
|
// preTaskIds used to create new connection |
||||||
|
preTasksToAdd: function () { |
||||||
|
let toAddTasks = this.preTasks.filter(taskId => { |
||||||
|
return (this.preTasksOld.indexOf(taskId) === -1) |
||||||
|
}) |
||||||
|
return toAddTasks |
||||||
|
}, |
||||||
|
// preTaskIds used to delete connection |
||||||
|
preTasksToDelete: function () { |
||||||
|
return this.preTasksOld.filter(taskId => this.preTasks.indexOf(taskId) === -1) |
||||||
|
}, |
||||||
|
}, |
||||||
|
methods: { |
||||||
|
// Pass data to parent-level to process dag |
||||||
|
_verification () { |
||||||
|
this.$emit('on-pre-tasks', { |
||||||
|
preTasks: this.preTasks, |
||||||
|
preTasksToAdd: this.preTasksToAdd, |
||||||
|
preTasksToDelete: this.preTasksToDelete, |
||||||
|
}) |
||||||
|
return true |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
</script> |
||||||
@ -0,0 +1,45 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.util; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.util.HashMap; |
||||||
|
|
||||||
|
import org.ho.yaml.Yaml; |
||||||
|
import org.springframework.core.io.DefaultResourceLoader; |
||||||
|
import org.springframework.core.io.Resource; |
||||||
|
|
||||||
|
/** |
||||||
|
* read yml file |
||||||
|
*/ |
||||||
|
public class YmlReader { |
||||||
|
public static HashMap<String,HashMap<String, String>> map; |
||||||
|
public String getDataYml(String filePath, String key1, String key2) { |
||||||
|
Yaml yaml = new Yaml(); |
||||||
|
Resource resource = new DefaultResourceLoader().getResource("classpath:" + filePath + ".yml"); |
||||||
|
try { |
||||||
|
InputStream inputStream = resource.getInputStream(); |
||||||
|
map = yaml.loadType(inputStream, HashMap.class); |
||||||
|
} catch (IOException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
String data = map.get(key1).get(key2); |
||||||
|
return data; |
||||||
|
} |
||||||
|
} |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue