Browse Source
* Add FileUtilsTest.java , the unit test for FileUtils (#1493) * #839 enhancement : add Spark Task Component can switch Spark Version (#1494) * add Spark Version in Spark Component add Spark Version in Spark Component * add license for SparkVersion.class add license * 1 add spark task UT 2 add spark version param check * add assert check for sparkTaskTest * fix AbstractTask's handle method exception (#1490) * fix AbstractTask's handle method exception * update ut * add ZkServer for UT (#1499) * add ZkServer for UT * Add FileUtilsTest.java , the unit test for FileUtils (#1493) (#1) * updates for reference ZkServer * DAG automatic layout (#1497) * Password verification and v-for add key * DAG automatic layout * Add common utils CollectionUtils.java DateUtils.java unit test (#1496) * dateutil test * pom.xml * Update README_zh_CN.md * Update README.md * Add ExcelUtilsTest.java , the unit test for ExcelUtils (#1500) * fix issue:1477 some tasks would be running all the time when db delayed(#1477) (#1501) * fix issue:1477 some tasks would be running all the time when db delayed * fix issue:1477 some tasks would be running all the time when db delayed * fix issue:1477 some tasks would be running all the time when db delayed * change npm run build:combined to npm burn build:release (#1504)pull/2/head
Tboy
5 years ago
committed by
GitHub
38 changed files with 927 additions and 141 deletions
@ -0,0 +1,92 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.alert.utils; |
||||
|
||||
import org.junit.After; |
||||
import org.junit.Before; |
||||
import org.junit.Rule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.ExpectedException; |
||||
import org.junit.rules.TemporaryFolder; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import java.io.File; |
||||
import static org.junit.Assert.assertTrue; |
||||
|
||||
public class ExcelUtilsTest { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ExcelUtilsTest.class); |
||||
|
||||
@Rule |
||||
public ExpectedException expectedException = ExpectedException.none(); |
||||
|
||||
@Rule |
||||
public TemporaryFolder folder = new TemporaryFolder(); |
||||
|
||||
private String rootPath = null; |
||||
|
||||
@Before |
||||
public void setUp() throws Exception { |
||||
|
||||
folder.create(); |
||||
rootPath = folder.getRoot().getAbsolutePath(); |
||||
} |
||||
|
||||
@After |
||||
public void tearDown() throws Exception { |
||||
|
||||
folder.delete(); |
||||
} |
||||
|
||||
/** |
||||
* Test GenExcelFile |
||||
*/ |
||||
@Test |
||||
public void testGenExcelFile() { |
||||
|
||||
//Define dest file path
|
||||
String xlsFilePath = rootPath + System.getProperty("file.separator"); |
||||
logger.info("xlsFilePath: "+xlsFilePath); |
||||
|
||||
//Define correctContent
|
||||
String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]"; |
||||
|
||||
//Define incorrectContent
|
||||
String incorrectContent1 = "{\"name\":\"ds name\",\"value\":\"ds value\"}"; |
||||
|
||||
//Define title
|
||||
String title = "test report"; |
||||
|
||||
//Invoke genExcelFile with correctContent
|
||||
ExcelUtils.genExcelFile(correctContent, title, xlsFilePath); |
||||
|
||||
//Test file exists
|
||||
File xlsFile = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); |
||||
assertTrue(xlsFile.exists()); |
||||
|
||||
//Expected RuntimeException
|
||||
expectedException.expect(RuntimeException.class); |
||||
|
||||
//Expected error message
|
||||
expectedException.expectMessage("itemsList is null"); |
||||
|
||||
//Invoke genExcelFile with incorrectContent, will cause RuntimeException
|
||||
ExcelUtils.genExcelFile(incorrectContent1, title, xlsFilePath); |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,40 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.enums; |
||||
|
||||
import com.baomidou.mybatisplus.annotation.EnumValue; |
||||
import lombok.Getter; |
||||
|
||||
@Getter |
||||
public enum SparkVersion { |
||||
|
||||
/** |
||||
* 0 SPARK1 |
||||
* 1 SPARK2 |
||||
*/ |
||||
SPARK1(0, "SPARK1"), |
||||
SPARK2(1, "SPARK2"); |
||||
|
||||
SparkVersion(int code, String descp){ |
||||
this.code = code; |
||||
this.descp = descp; |
||||
} |
||||
|
||||
@EnumValue |
||||
private final int code; |
||||
private final String descp; |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.zk; |
||||
|
||||
import org.junit.After; |
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
/** |
||||
* demo for using zkServer |
||||
*/ |
||||
public class TestZk { |
||||
|
||||
@Before |
||||
public void before(){ |
||||
ZKServer.start(); |
||||
} |
||||
|
||||
@Test |
||||
public void test(){ |
||||
Assert.assertTrue(ZKServer.isStarted()); |
||||
} |
||||
|
||||
@After |
||||
public void after(){ |
||||
ZKServer.stop(); |
||||
} |
||||
} |
||||
@ -0,0 +1,141 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.worker.task.spark; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
import org.apache.dolphinscheduler.common.enums.SparkVersion; |
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils; |
||||
import org.apache.dolphinscheduler.server.utils.ParamUtils; |
||||
import org.apache.dolphinscheduler.server.utils.SparkArgsUtils; |
||||
import org.apache.dolphinscheduler.server.worker.task.TaskProps; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
|
||||
public class SparkTaskTest { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SparkTaskTest.class); |
||||
|
||||
/** |
||||
* spark1 command |
||||
*/ |
||||
private static final String SPARK1_COMMAND = "${SPARK_HOME1}/bin/spark-submit"; |
||||
|
||||
/** |
||||
* spark2 command |
||||
*/ |
||||
private static final String SPARK2_COMMAND = "${SPARK_HOME2}/bin/spark-submit"; |
||||
|
||||
@Test |
||||
public void testSparkTaskInit() { |
||||
|
||||
TaskProps taskProps = new TaskProps(); |
||||
|
||||
String spark1Params = "{" + |
||||
"\"mainArgs\":\"\", " + |
||||
"\"driverMemory\":\"1G\", " + |
||||
"\"executorMemory\":\"2G\", " + |
||||
"\"programType\":\"SCALA\", " + |
||||
"\"mainClass\":\"basicetl.GlobalUserCar\", " + |
||||
"\"driverCores\":\"2\", " + |
||||
"\"deployMode\":\"cluster\", " + |
||||
"\"executorCores\":2, " + |
||||
"\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, " + |
||||
"\"sparkVersion\":\"SPARK1\", " + |
||||
"\"numExecutors\":\"10\", " + |
||||
"\"localParams\":[], " + |
||||
"\"others\":\"\", " + |
||||
"\"resourceList\":[]" + |
||||
"}"; |
||||
|
||||
String spark2Params = "{" + |
||||
"\"mainArgs\":\"\", " + |
||||
"\"driverMemory\":\"1G\", " + |
||||
"\"executorMemory\":\"2G\", " + |
||||
"\"programType\":\"SCALA\", " + |
||||
"\"mainClass\":\"basicetl.GlobalUserCar\", " + |
||||
"\"driverCores\":\"2\", " + |
||||
"\"deployMode\":\"cluster\", " + |
||||
"\"executorCores\":2, " + |
||||
"\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, " + |
||||
"\"sparkVersion\":\"SPARK2\", " + |
||||
"\"numExecutors\":\"10\", " + |
||||
"\"localParams\":[], " + |
||||
"\"others\":\"\", " + |
||||
"\"resourceList\":[]" + |
||||
"}"; |
||||
|
||||
taskProps.setTaskParams(spark2Params); |
||||
|
||||
logger.info("spark task params {}", taskProps.getTaskParams()); |
||||
|
||||
SparkParameters sparkParameters = JSONUtils.parseObject(taskProps.getTaskParams(), SparkParameters.class); |
||||
|
||||
assert sparkParameters != null; |
||||
if (!sparkParameters.checkParameters()) { |
||||
throw new RuntimeException("spark task params is not valid"); |
||||
} |
||||
sparkParameters.setQueue(taskProps.getQueue()); |
||||
|
||||
if (StringUtils.isNotEmpty(sparkParameters.getMainArgs())) { |
||||
String args = sparkParameters.getMainArgs(); |
||||
|
||||
/** |
||||
* combining local and global parameters |
||||
*/ |
||||
Map<String, Property> paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), |
||||
taskProps.getDefinedParams(), |
||||
sparkParameters.getLocalParametersMap(), |
||||
taskProps.getCmdTypeIfComplement(), |
||||
taskProps.getScheduleTime()); |
||||
if (paramsMap != null) { |
||||
args = ParameterUtils.convertParameterPlaceholders(args, ParamUtils.convert(paramsMap)); |
||||
} |
||||
sparkParameters.setMainArgs(args); |
||||
} |
||||
|
||||
List<String> args = new ArrayList<>(); |
||||
|
||||
//spark version
|
||||
String sparkCommand = SPARK2_COMMAND; |
||||
|
||||
if (SparkVersion.SPARK1.name().equals(sparkParameters.getSparkVersion())) { |
||||
sparkCommand = SPARK1_COMMAND; |
||||
} |
||||
|
||||
args.add(sparkCommand); |
||||
|
||||
// other parameters
|
||||
args.addAll(SparkArgsUtils.buildArgs(sparkParameters)); |
||||
|
||||
String sparkArgs = String.join(" ", args); |
||||
|
||||
logger.info("spark task command : {}", sparkArgs); |
||||
|
||||
Assert.assertEquals(sparkArgs.split(" ")[0], SPARK2_COMMAND ); |
||||
|
||||
} |
||||
} |
||||
Loading…
Reference in new issue