Shiwen Cheng
4 years ago
committed by
GitHub
15 changed files with 368 additions and 188 deletions
@ -0,0 +1,85 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ProgramType; |
||||||
|
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||||
|
import org.apache.dolphinscheduler.common.task.mr.MapReduceParameters; |
||||||
|
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* mapreduce args utils |
||||||
|
*/ |
||||||
|
public class MapReduceArgsUtils { |
||||||
|
|
||||||
|
private MapReduceArgsUtils() { |
||||||
|
throw new IllegalStateException("Utility class"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* build args |
||||||
|
* |
||||||
|
* @param param param |
||||||
|
* @return argument list |
||||||
|
*/ |
||||||
|
public static List<String> buildArgs(MapReduceParameters param) { |
||||||
|
List<String> args = new ArrayList<>(); |
||||||
|
|
||||||
|
ResourceInfo mainJar = param.getMainJar(); |
||||||
|
if (mainJar != null) { |
||||||
|
args.add(Constants.JAR); |
||||||
|
args.add(mainJar.getRes()); |
||||||
|
} |
||||||
|
|
||||||
|
ProgramType programType = param.getProgramType(); |
||||||
|
String mainClass = param.getMainClass(); |
||||||
|
if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { |
||||||
|
args.add(mainClass); |
||||||
|
} |
||||||
|
|
||||||
|
String appName = param.getAppName(); |
||||||
|
if (StringUtils.isNotEmpty(appName)) { |
||||||
|
args.add(String.format("%s%s=%s", Constants.D, Constants.MR_NAME, ArgsUtils.escape(appName))); |
||||||
|
} |
||||||
|
|
||||||
|
String others = param.getOthers(); |
||||||
|
if (StringUtils.isEmpty(others) || !others.contains(Constants.MR_QUEUE)) { |
||||||
|
String queue = param.getQueue(); |
||||||
|
if (StringUtils.isNotEmpty(queue)) { |
||||||
|
args.add(String.format("%s%s=%s", Constants.D, Constants.MR_QUEUE, queue)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// -conf -archives -files -libjars -D
|
||||||
|
if (StringUtils.isNotEmpty(others)) { |
||||||
|
args.add(others); |
||||||
|
} |
||||||
|
|
||||||
|
String mainArgs = param.getMainArgs(); |
||||||
|
if (StringUtils.isNotEmpty(mainArgs)) { |
||||||
|
args.add(mainArgs); |
||||||
|
} |
||||||
|
|
||||||
|
return args; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,95 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.utils; |
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ProgramType; |
||||||
|
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||||
|
import org.apache.dolphinscheduler.common.task.mr.MapReduceParameters; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* Test MapReduceArgsUtils |
||||||
|
*/ |
||||||
|
public class MapReduceArgsUtilsTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(MapReduceArgsUtilsTest.class); |
||||||
|
|
||||||
|
public String mainClass = "com.examples.WordCount"; |
||||||
|
public ResourceInfo mainJar = null; |
||||||
|
public String mainArgs = "/user/joe/wordcount/input /user/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt"; |
||||||
|
public ProgramType programType = ProgramType.JAVA; |
||||||
|
public String others = "-files cachefile.txt -libjars mylib.jar -archives myarchive.zip -Dwordcount.case.sensitive=false"; |
||||||
|
public String appName = "mapreduce test"; |
||||||
|
public String queue = "queue1"; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void setUp() { |
||||||
|
ResourceInfo main = new ResourceInfo(); |
||||||
|
main.setRes("testspark-1.0.0-SNAPSHOT.jar"); |
||||||
|
mainJar = main; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test buildArgs |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testBuildArgs() { |
||||||
|
//Define params
|
||||||
|
MapReduceParameters param = new MapReduceParameters(); |
||||||
|
param.setMainClass(mainClass); |
||||||
|
param.setMainJar(mainJar); |
||||||
|
param.setMainArgs(mainArgs); |
||||||
|
param.setProgramType(programType); |
||||||
|
param.setOthers(others); |
||||||
|
param.setAppName(appName); |
||||||
|
param.setQueue(queue); |
||||||
|
|
||||||
|
//Invoke buildArgs
|
||||||
|
List<String> result = MapReduceArgsUtils.buildArgs(param); |
||||||
|
for (String s : result) { |
||||||
|
logger.info(s); |
||||||
|
} |
||||||
|
|
||||||
|
//Expected values and order
|
||||||
|
assertEquals(7, result.size()); |
||||||
|
|
||||||
|
assertEquals("jar", result.get(0)); |
||||||
|
assertEquals(mainJar.getRes(), result.get(1)); |
||||||
|
assertEquals(mainClass, result.get(2)); |
||||||
|
assertEquals(String.format("-D%s=%s", Constants.MR_NAME, ArgsUtils.escape(appName)), result.get(3)); |
||||||
|
assertEquals(String.format("-D%s=%s", Constants.MR_QUEUE, queue), result.get(4)); |
||||||
|
assertEquals(others, result.get(5)); |
||||||
|
assertEquals(mainArgs, result.get(6)); |
||||||
|
|
||||||
|
//Others param without --queue
|
||||||
|
param.setOthers("-files xxx/hive-site.xml"); |
||||||
|
param.setQueue(null); |
||||||
|
result = MapReduceArgsUtils.buildArgs(param); |
||||||
|
assertEquals(6, result.size()); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
Loading…
Reference in new issue