Browse Source
* Add FileUtilsTest.java , the unit test for FileUtils (#1493) * #839 enhancement : add Spark Task Component can switch Spark Version (#1494) * add Spark Version in Spark Component add Spark Version in Spark Component * add license for SparkVersion.class add license * 1 add spark task UT 2 add spark version param check * add assert check for sparkTaskTest * fix AbstractTask's handle method exception (#1490) * fix AbstractTask's handle method exception * update ut * add ZkServer for UT (#1499) * add ZkServer for UT * Add FileUtilsTest.java , the unit test for FileUtils (#1493) (#1) * updates for reference ZkServer * DAG automatic layout (#1497) * Password verification and v-for add key * DAG automatic layout * Add common utils CollectionUtils.java DateUtils.java unit test (#1496) * dateutil test * pom.xml * Update README_zh_CN.md * Update README.md * Add ExcelUtilsTest.java , the unit test for ExcelUtils (#1500) * fix issue:1477 some tasks would be running all the time when db delayed(#1477) (#1501) * fix issue:1477 some tasks would be running all the time when db delayed * fix issue:1477 some tasks would be running all the time when db delayed * fix issue:1477 some tasks would be running all the time when db delayed * change npm run build:combined to npm burn build:release (#1504)pull/2/head
Tboy
5 years ago
committed by
GitHub
38 changed files with 927 additions and 141 deletions
@ -0,0 +1,92 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.alert.utils; |
||||||
|
|
||||||
|
import org.junit.After; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.ExpectedException; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import java.io.File; |
||||||
|
import static org.junit.Assert.assertTrue; |
||||||
|
|
||||||
|
public class ExcelUtilsTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ExcelUtilsTest.class); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public ExpectedException expectedException = ExpectedException.none(); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public TemporaryFolder folder = new TemporaryFolder(); |
||||||
|
|
||||||
|
private String rootPath = null; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void setUp() throws Exception { |
||||||
|
|
||||||
|
folder.create(); |
||||||
|
rootPath = folder.getRoot().getAbsolutePath(); |
||||||
|
} |
||||||
|
|
||||||
|
@After |
||||||
|
public void tearDown() throws Exception { |
||||||
|
|
||||||
|
folder.delete(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test GenExcelFile |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testGenExcelFile() { |
||||||
|
|
||||||
|
//Define dest file path
|
||||||
|
String xlsFilePath = rootPath + System.getProperty("file.separator"); |
||||||
|
logger.info("xlsFilePath: "+xlsFilePath); |
||||||
|
|
||||||
|
//Define correctContent
|
||||||
|
String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]"; |
||||||
|
|
||||||
|
//Define incorrectContent
|
||||||
|
String incorrectContent1 = "{\"name\":\"ds name\",\"value\":\"ds value\"}"; |
||||||
|
|
||||||
|
//Define title
|
||||||
|
String title = "test report"; |
||||||
|
|
||||||
|
//Invoke genExcelFile with correctContent
|
||||||
|
ExcelUtils.genExcelFile(correctContent, title, xlsFilePath); |
||||||
|
|
||||||
|
//Test file exists
|
||||||
|
File xlsFile = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); |
||||||
|
assertTrue(xlsFile.exists()); |
||||||
|
|
||||||
|
//Expected RuntimeException
|
||||||
|
expectedException.expect(RuntimeException.class); |
||||||
|
|
||||||
|
//Expected error message
|
||||||
|
expectedException.expectMessage("itemsList is null"); |
||||||
|
|
||||||
|
//Invoke genExcelFile with incorrectContent, will cause RuntimeException
|
||||||
|
ExcelUtils.genExcelFile(incorrectContent1, title, xlsFilePath); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,40 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.enums; |
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.EnumValue; |
||||||
|
import lombok.Getter; |
||||||
|
|
||||||
|
@Getter |
||||||
|
public enum SparkVersion { |
||||||
|
|
||||||
|
/** |
||||||
|
* 0 SPARK1 |
||||||
|
* 1 SPARK2 |
||||||
|
*/ |
||||||
|
SPARK1(0, "SPARK1"), |
||||||
|
SPARK2(1, "SPARK2"); |
||||||
|
|
||||||
|
SparkVersion(int code, String descp){ |
||||||
|
this.code = code; |
||||||
|
this.descp = descp; |
||||||
|
} |
||||||
|
|
||||||
|
@EnumValue |
||||||
|
private final int code; |
||||||
|
private final String descp; |
||||||
|
} |
||||||
@ -0,0 +1,43 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import org.junit.After; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
/** |
||||||
|
* demo for using zkServer |
||||||
|
*/ |
||||||
|
public class TestZk { |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before(){ |
||||||
|
ZKServer.start(); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void test(){ |
||||||
|
Assert.assertTrue(ZKServer.isStarted()); |
||||||
|
} |
||||||
|
|
||||||
|
@After |
||||||
|
public void after(){ |
||||||
|
ZKServer.stop(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,141 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.server.worker.task.spark; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.apache.dolphinscheduler.common.enums.SparkVersion; |
||||||
|
import org.apache.dolphinscheduler.common.process.Property; |
||||||
|
import org.apache.dolphinscheduler.common.task.spark.SparkParameters; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.ParameterUtils; |
||||||
|
import org.apache.dolphinscheduler.server.utils.ParamUtils; |
||||||
|
import org.apache.dolphinscheduler.server.utils.SparkArgsUtils; |
||||||
|
import org.apache.dolphinscheduler.server.worker.task.TaskProps; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
|
||||||
|
public class SparkTaskTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SparkTaskTest.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* spark1 command |
||||||
|
*/ |
||||||
|
private static final String SPARK1_COMMAND = "${SPARK_HOME1}/bin/spark-submit"; |
||||||
|
|
||||||
|
/** |
||||||
|
* spark2 command |
||||||
|
*/ |
||||||
|
private static final String SPARK2_COMMAND = "${SPARK_HOME2}/bin/spark-submit"; |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSparkTaskInit() { |
||||||
|
|
||||||
|
TaskProps taskProps = new TaskProps(); |
||||||
|
|
||||||
|
String spark1Params = "{" + |
||||||
|
"\"mainArgs\":\"\", " + |
||||||
|
"\"driverMemory\":\"1G\", " + |
||||||
|
"\"executorMemory\":\"2G\", " + |
||||||
|
"\"programType\":\"SCALA\", " + |
||||||
|
"\"mainClass\":\"basicetl.GlobalUserCar\", " + |
||||||
|
"\"driverCores\":\"2\", " + |
||||||
|
"\"deployMode\":\"cluster\", " + |
||||||
|
"\"executorCores\":2, " + |
||||||
|
"\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, " + |
||||||
|
"\"sparkVersion\":\"SPARK1\", " + |
||||||
|
"\"numExecutors\":\"10\", " + |
||||||
|
"\"localParams\":[], " + |
||||||
|
"\"others\":\"\", " + |
||||||
|
"\"resourceList\":[]" + |
||||||
|
"}"; |
||||||
|
|
||||||
|
String spark2Params = "{" + |
||||||
|
"\"mainArgs\":\"\", " + |
||||||
|
"\"driverMemory\":\"1G\", " + |
||||||
|
"\"executorMemory\":\"2G\", " + |
||||||
|
"\"programType\":\"SCALA\", " + |
||||||
|
"\"mainClass\":\"basicetl.GlobalUserCar\", " + |
||||||
|
"\"driverCores\":\"2\", " + |
||||||
|
"\"deployMode\":\"cluster\", " + |
||||||
|
"\"executorCores\":2, " + |
||||||
|
"\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, " + |
||||||
|
"\"sparkVersion\":\"SPARK2\", " + |
||||||
|
"\"numExecutors\":\"10\", " + |
||||||
|
"\"localParams\":[], " + |
||||||
|
"\"others\":\"\", " + |
||||||
|
"\"resourceList\":[]" + |
||||||
|
"}"; |
||||||
|
|
||||||
|
taskProps.setTaskParams(spark2Params); |
||||||
|
|
||||||
|
logger.info("spark task params {}", taskProps.getTaskParams()); |
||||||
|
|
||||||
|
SparkParameters sparkParameters = JSONUtils.parseObject(taskProps.getTaskParams(), SparkParameters.class); |
||||||
|
|
||||||
|
assert sparkParameters != null; |
||||||
|
if (!sparkParameters.checkParameters()) { |
||||||
|
throw new RuntimeException("spark task params is not valid"); |
||||||
|
} |
||||||
|
sparkParameters.setQueue(taskProps.getQueue()); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(sparkParameters.getMainArgs())) { |
||||||
|
String args = sparkParameters.getMainArgs(); |
||||||
|
|
||||||
|
/** |
||||||
|
* combining local and global parameters |
||||||
|
*/ |
||||||
|
Map<String, Property> paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), |
||||||
|
taskProps.getDefinedParams(), |
||||||
|
sparkParameters.getLocalParametersMap(), |
||||||
|
taskProps.getCmdTypeIfComplement(), |
||||||
|
taskProps.getScheduleTime()); |
||||||
|
if (paramsMap != null) { |
||||||
|
args = ParameterUtils.convertParameterPlaceholders(args, ParamUtils.convert(paramsMap)); |
||||||
|
} |
||||||
|
sparkParameters.setMainArgs(args); |
||||||
|
} |
||||||
|
|
||||||
|
List<String> args = new ArrayList<>(); |
||||||
|
|
||||||
|
//spark version
|
||||||
|
String sparkCommand = SPARK2_COMMAND; |
||||||
|
|
||||||
|
if (SparkVersion.SPARK1.name().equals(sparkParameters.getSparkVersion())) { |
||||||
|
sparkCommand = SPARK1_COMMAND; |
||||||
|
} |
||||||
|
|
||||||
|
args.add(sparkCommand); |
||||||
|
|
||||||
|
// other parameters
|
||||||
|
args.addAll(SparkArgsUtils.buildArgs(sparkParameters)); |
||||||
|
|
||||||
|
String sparkArgs = String.join(" ", args); |
||||||
|
|
||||||
|
logger.info("spark task command : {}", sparkArgs); |
||||||
|
|
||||||
|
Assert.assertEquals(sparkArgs.split(" ")[0], SPARK2_COMMAND ); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue