Paul Zhang
2 years ago
committed by
GitHub
11 changed files with 641 additions and 439 deletions
@ -0,0 +1,103 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.flink; |
||||
|
||||
import org.apache.commons.lang3.SystemUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.StandardOpenOption; |
||||
import java.nio.file.attribute.FileAttribute; |
||||
import java.nio.file.attribute.PosixFilePermission; |
||||
import java.nio.file.attribute.PosixFilePermissions; |
||||
import java.util.Set; |
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; |
||||
|
||||
public class FileUtils { |
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); |
||||
private FileUtils() {} |
||||
|
||||
public static String getInitScriptFilePath(TaskExecutionContext taskExecutionContext) { |
||||
return String.format("%s/%s_init.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); |
||||
} |
||||
|
||||
public static String getScriptFilePath(TaskExecutionContext taskExecutionContext) { |
||||
return String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); |
||||
} |
||||
|
||||
public static void generateScriptFile(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { |
||||
String initScriptFilePath = FileUtils.getInitScriptFilePath(taskExecutionContext); |
||||
String scriptFilePath = FileUtils.getScriptFilePath(taskExecutionContext); |
||||
String initOptionsString = StringUtils.join( |
||||
FlinkArgsUtils.buildInitOptionsForSql(flinkParameters), |
||||
FlinkConstants.FLINK_SQL_NEWLINE |
||||
).concat(FlinkConstants.FLINK_SQL_NEWLINE); |
||||
writeScriptFile(initScriptFilePath, initOptionsString + flinkParameters.getInitScript()); |
||||
writeScriptFile(scriptFilePath, flinkParameters.getRawScript()); |
||||
} |
||||
|
||||
private static void writeScriptFile(String scriptFileFullPath, String script) { |
||||
File scriptFile = new File(scriptFileFullPath); |
||||
Path path = scriptFile.toPath(); |
||||
if (Files.exists(path)) { |
||||
try { |
||||
Files.delete(path); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(String.format("Flink Script file exists in path: %s before creation and cannot be deleted", path), e); |
||||
} |
||||
} |
||||
|
||||
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(RWXR_XR_X); |
||||
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms); |
||||
try { |
||||
if (SystemUtils.IS_OS_WINDOWS) { |
||||
Files.createFile(path); |
||||
} else { |
||||
if (!scriptFile.getParentFile().exists()) { |
||||
scriptFile.getParentFile().mkdirs(); |
||||
} |
||||
Files.createFile(path, attr); |
||||
} |
||||
|
||||
if (StringUtils.isNotEmpty(script)) { |
||||
String replacedScript = script.replaceAll("\\r\\n", "\n"); |
||||
FileUtils.writeStringToFile(scriptFile, replacedScript, StandardOpenOption.APPEND); |
||||
} |
||||
} catch (IOException e) { |
||||
throw new RuntimeException("Generate flink SQL script error", e); |
||||
} |
||||
} |
||||
|
||||
private static void writeStringToFile(File file, String content, StandardOpenOption standardOpenOption) { |
||||
try { |
||||
LOGGER.info("Writing content: " + content); |
||||
LOGGER.info("To file: " + file.getAbsolutePath()); |
||||
Files.write(file.getAbsoluteFile().toPath(), content.getBytes(StandardCharsets.UTF_8), standardOpenOption); |
||||
} catch(IOException e) { |
||||
throw new RuntimeException("Error writing file: " + file.getAbsoluteFile(), e); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,271 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.flink; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; |
||||
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Optional; |
||||
|
||||
/** |
||||
* flink args utils |
||||
*/ |
||||
public class FlinkArgsUtils { |
||||
|
||||
private FlinkArgsUtils() { |
||||
throw new IllegalStateException("Utility class"); |
||||
} |
||||
|
||||
private static final String LOCAL_DEPLOY_MODE = "local"; |
||||
private static final String FLINK_VERSION_BEFORE_1_10 = "<1.10"; |
||||
private static final String FLINK_VERSION_AFTER_OR_EQUALS_1_12 = ">=1.12"; |
||||
private static final String FLINK_VERSION_AFTER_OR_EQUALS_1_13 = ">=1.13"; |
||||
/** |
||||
* default flink deploy mode |
||||
*/ |
||||
public static final FlinkDeployMode DEFAULT_DEPLOY_MODE = FlinkDeployMode.CLUSTER; |
||||
|
||||
/** |
||||
* build flink command line |
||||
* |
||||
* @param param flink parameters |
||||
* @return argument list |
||||
*/ |
||||
public static List<String> buildCommandLine(TaskExecutionContext taskExecutionContext, FlinkParameters param) { |
||||
switch (param.getProgramType()) { |
||||
case SQL: |
||||
return buildCommandLineForSql(taskExecutionContext, param); |
||||
default: |
||||
return buildCommandLineForOthers(taskExecutionContext, param); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* build flink command line for SQL |
||||
* |
||||
* @return argument list |
||||
*/ |
||||
private static List<String> buildCommandLineForSql(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { |
||||
List<String> args = new ArrayList<>(); |
||||
|
||||
args.add(FlinkConstants.FLINK_SQL_COMMAND); |
||||
|
||||
// -i
|
||||
String initScriptFilePath = FileUtils.getInitScriptFilePath(taskExecutionContext); |
||||
args.add(FlinkConstants.FLINK_SQL_INIT_FILE); |
||||
args.add(initScriptFilePath); |
||||
|
||||
// -f
|
||||
String scriptFilePath = FileUtils.getScriptFilePath(taskExecutionContext); |
||||
args.add(FlinkConstants.FLINK_SQL_SCRIPT_FILE); |
||||
args.add(scriptFilePath); |
||||
|
||||
String others = flinkParameters.getOthers(); |
||||
if (StringUtils.isNotEmpty(others)) { |
||||
args.add(others); |
||||
} |
||||
return args; |
||||
} |
||||
|
||||
public static List<String> buildInitOptionsForSql(FlinkParameters flinkParameters) { |
||||
List<String> initOptions = new ArrayList<>(); |
||||
|
||||
FlinkDeployMode deployMode = Optional.ofNullable(flinkParameters.getDeployMode()).orElse(FlinkDeployMode.CLUSTER); |
||||
|
||||
/** |
||||
* Currently flink sql on yarn only supports yarn-per-job mode |
||||
*/ |
||||
if (FlinkDeployMode.CLUSTER == deployMode) { |
||||
// execution.target
|
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, "local")); |
||||
} else { |
||||
// execution.target
|
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, FlinkConstants.FLINK_YARN_PER_JOB)); |
||||
|
||||
// taskmanager.numberOfTaskSlots
|
||||
int slot = flinkParameters.getSlot(); |
||||
if (slot > 0) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_NUMBEROFTASKSLOTS, slot)); |
||||
} |
||||
|
||||
// yarn.application.name
|
||||
String appName = flinkParameters.getAppName(); |
||||
if (StringUtils.isNotEmpty(appName)) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_NAME, ArgsUtils.escape(appName))); |
||||
} |
||||
|
||||
// jobmanager.memory.process.size
|
||||
String jobManagerMemory = flinkParameters.getJobManagerMemory(); |
||||
if (StringUtils.isNotEmpty(jobManagerMemory)) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_JOBMANAGER_MEMORY_PROCESS_SIZE, jobManagerMemory)); |
||||
} |
||||
|
||||
// taskmanager.memory.process.size
|
||||
String taskManagerMemory = flinkParameters.getTaskManagerMemory(); |
||||
if (StringUtils.isNotEmpty(taskManagerMemory)) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_MEMORY_PROCESS_SIZE, taskManagerMemory)); |
||||
} |
||||
|
||||
// yarn.application.queue
|
||||
String others = flinkParameters.getOthers(); |
||||
if (StringUtils.isEmpty(others) || !others.contains(FlinkConstants.FLINK_QUEUE)) { |
||||
String queue = flinkParameters.getQueue(); |
||||
if (StringUtils.isNotEmpty(queue)) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_QUEUE, queue)); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// parallelism.default
|
||||
int parallelism = flinkParameters.getParallelism(); |
||||
if (parallelism > 0) { |
||||
initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_PARALLELISM_DEFAULT, parallelism)); |
||||
} |
||||
|
||||
return initOptions; |
||||
} |
||||
|
||||
private static List<String> buildCommandLineForOthers(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { |
||||
List<String> args = new ArrayList<>(); |
||||
|
||||
args.add(FlinkConstants.FLINK_COMMAND); |
||||
FlinkDeployMode deployMode = Optional.ofNullable(flinkParameters.getDeployMode()).orElse(DEFAULT_DEPLOY_MODE); |
||||
String flinkVersion = flinkParameters.getFlinkVersion(); |
||||
// build run command
|
||||
switch (deployMode) { |
||||
case CLUSTER: |
||||
if (FLINK_VERSION_AFTER_OR_EQUALS_1_12.equals(flinkVersion) || FLINK_VERSION_AFTER_OR_EQUALS_1_13.equals(flinkVersion)) { |
||||
args.add(FlinkConstants.FLINK_RUN); //run
|
||||
args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t
|
||||
args.add(FlinkConstants.FLINK_YARN_PER_JOB); //yarn-per-job
|
||||
} else { |
||||
args.add(FlinkConstants.FLINK_RUN); //run
|
||||
args.add(FlinkConstants.FLINK_RUN_MODE); //-m
|
||||
args.add(FlinkConstants.FLINK_YARN_CLUSTER); //yarn-cluster
|
||||
} |
||||
break; |
||||
case APPLICATION: |
||||
args.add(FlinkConstants.FLINK_RUN_APPLICATION); //run-application
|
||||
args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t
|
||||
args.add(FlinkConstants.FLINK_YARN_APPLICATION); //yarn-application
|
||||
break; |
||||
case LOCAL: |
||||
args.add(FlinkConstants.FLINK_RUN); //run
|
||||
break; |
||||
} |
||||
|
||||
String others = flinkParameters.getOthers(); |
||||
|
||||
// build args
|
||||
switch (deployMode) { |
||||
case CLUSTER: |
||||
case APPLICATION: |
||||
int slot = flinkParameters.getSlot(); |
||||
if (slot > 0) { |
||||
args.add(FlinkConstants.FLINK_YARN_SLOT); |
||||
args.add(String.format("%d", slot)); //-ys
|
||||
} |
||||
|
||||
String appName = flinkParameters.getAppName(); |
||||
if (StringUtils.isNotEmpty(appName)) { //-ynm
|
||||
args.add(FlinkConstants.FLINK_APP_NAME); |
||||
args.add(ArgsUtils.escape(appName)); |
||||
} |
||||
|
||||
// judge flink version, the parameter -yn has removed from flink 1.10
|
||||
if (flinkVersion == null || FLINK_VERSION_BEFORE_1_10.equals(flinkVersion)) { |
||||
int taskManager = flinkParameters.getTaskManager(); |
||||
if (taskManager > 0) { //-yn
|
||||
args.add(FlinkConstants.FLINK_TASK_MANAGE); |
||||
args.add(String.format("%d", taskManager)); |
||||
} |
||||
} |
||||
String jobManagerMemory = flinkParameters.getJobManagerMemory(); |
||||
if (StringUtils.isNotEmpty(jobManagerMemory)) { |
||||
args.add(FlinkConstants.FLINK_JOB_MANAGE_MEM); |
||||
args.add(jobManagerMemory); //-yjm
|
||||
} |
||||
|
||||
String taskManagerMemory = flinkParameters.getTaskManagerMemory(); |
||||
if (StringUtils.isNotEmpty(taskManagerMemory)) { // -ytm
|
||||
args.add(FlinkConstants.FLINK_TASK_MANAGE_MEM); |
||||
args.add(taskManagerMemory); |
||||
} |
||||
|
||||
if (StringUtils.isEmpty(others) || !others.contains(FlinkConstants.FLINK_QUEUE)) { |
||||
String queue = flinkParameters.getQueue(); |
||||
if (StringUtils.isNotEmpty(queue)) { // -yqu
|
||||
args.add(FlinkConstants.FLINK_QUEUE); |
||||
args.add(queue); |
||||
} |
||||
} |
||||
break; |
||||
case LOCAL: |
||||
break; |
||||
} |
||||
|
||||
int parallelism = flinkParameters.getParallelism(); |
||||
if (parallelism > 0) { |
||||
args.add(FlinkConstants.FLINK_PARALLELISM); |
||||
args.add(String.format("%d", parallelism)); // -p
|
||||
} |
||||
|
||||
// If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly
|
||||
// The task status will be synchronized with the cluster job status
|
||||
args.add(FlinkConstants.FLINK_SHUTDOWN_ON_ATTACHED_EXIT); // -sae
|
||||
|
||||
// -s -yqu -yat -yD -D
|
||||
if (StringUtils.isNotEmpty(others)) { |
||||
args.add(others); |
||||
} |
||||
|
||||
ProgramType programType = flinkParameters.getProgramType(); |
||||
String mainClass = flinkParameters.getMainClass(); |
||||
if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { |
||||
args.add(FlinkConstants.FLINK_MAIN_CLASS); //-c
|
||||
args.add(flinkParameters.getMainClass()); //main class
|
||||
} |
||||
|
||||
ResourceInfo mainJar = flinkParameters.getMainJar(); |
||||
if (mainJar != null) { |
||||
// -py
|
||||
if(ProgramType.PYTHON == programType) { |
||||
args.add(FlinkConstants.FLINK_PYTHON); |
||||
} |
||||
args.add(mainJar.getRes()); |
||||
} |
||||
|
||||
String mainArgs = flinkParameters.getMainArgs(); |
||||
if (StringUtils.isNotEmpty(mainArgs)) { |
||||
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); |
||||
args.add(ParameterUtils.convertParameterPlaceholders(mainArgs, ParamUtils.convert(paramsMap))); |
||||
} |
||||
|
||||
return args; |
||||
} |
||||
} |
||||
@ -0,0 +1,32 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.flink; |
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty; |
||||
|
||||
/** |
||||
* Flink deploy mode |
||||
*/ |
||||
public enum FlinkDeployMode { |
||||
@JsonProperty("local") |
||||
LOCAL, |
||||
@JsonProperty("cluster") |
||||
CLUSTER, |
||||
@JsonProperty("application") |
||||
APPLICATION |
||||
} |
||||
@ -0,0 +1,132 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.flink; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.List; |
||||
|
||||
public class FlinkArgsUtilsTest { |
||||
|
||||
private String joinStringListWithSpace(List<String> stringList) { |
||||
return String.join(" ", stringList); |
||||
} |
||||
|
||||
private FlinkParameters buildTestFlinkParametersWithDeployMode(FlinkDeployMode flinkDeployMode) { |
||||
FlinkParameters flinkParameters = new FlinkParameters(); |
||||
flinkParameters.setProgramType(ProgramType.SCALA); |
||||
flinkParameters.setDeployMode(flinkDeployMode); |
||||
flinkParameters.setParallelism(4); |
||||
ResourceInfo resourceInfo = new ResourceInfo(); |
||||
resourceInfo.setId(1); |
||||
resourceInfo.setResourceName("job"); |
||||
resourceInfo.setRes("/opt/job.jar"); |
||||
flinkParameters.setMainJar(resourceInfo); |
||||
flinkParameters.setMainClass("org.example.Main"); |
||||
flinkParameters.setSlot(4); |
||||
flinkParameters.setAppName("demo-app-name"); |
||||
flinkParameters.setJobManagerMemory("1024m"); |
||||
flinkParameters.setTaskManagerMemory("1024m"); |
||||
|
||||
return flinkParameters; |
||||
} |
||||
private TaskExecutionContext buildTestTaskExecutionContext() { |
||||
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); |
||||
taskExecutionContext.setTaskAppId("app-id"); |
||||
taskExecutionContext.setExecutePath("/tmp/execution"); |
||||
return taskExecutionContext; |
||||
} |
||||
|
||||
@Test |
||||
public void testRunJarInApplicationMode() throws Exception { |
||||
FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.APPLICATION); |
||||
List<String> commandLine = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals( |
||||
"flink run-application -t yarn-application -ys 4 -ynm demo-app-name -yjm 1024m -ytm 1024m -p 4 -sae -c org.example.Main /opt/job.jar", |
||||
joinStringListWithSpace(commandLine)); |
||||
} |
||||
|
||||
@Test |
||||
public void testRunJarInClusterMode() throws Exception { |
||||
FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.CLUSTER); |
||||
flinkParameters.setFlinkVersion("1.11"); |
||||
List<String> commandLine1 = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals( |
||||
"flink run -m yarn-cluster -ys 4 -ynm demo-app-name -yjm 1024m -ytm 1024m -p 4 -sae -c org.example.Main /opt/job.jar", |
||||
joinStringListWithSpace(commandLine1)); |
||||
|
||||
flinkParameters.setFlinkVersion("<1.10"); |
||||
List<String> commandLine2 = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals( |
||||
"flink run -m yarn-cluster -ys 4 -ynm demo-app-name -yjm 1024m -ytm 1024m -p 4 -sae -c org.example.Main /opt/job.jar", |
||||
joinStringListWithSpace(commandLine2)); |
||||
|
||||
flinkParameters.setFlinkVersion(">=1.12"); |
||||
List<String> commandLine3 = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals( |
||||
"flink run -t yarn-per-job -ys 4 -ynm demo-app-name -yjm 1024m -ytm 1024m -p 4 -sae -c org.example.Main /opt/job.jar", |
||||
joinStringListWithSpace(commandLine3)); |
||||
} |
||||
|
||||
@Test |
||||
public void testRunJarInLocalMode() throws Exception { |
||||
FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.LOCAL); |
||||
List<String> commandLine = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals( |
||||
"flink run -p 4 -sae -c org.example.Main /opt/job.jar", |
||||
joinStringListWithSpace(commandLine)); |
||||
} |
||||
|
||||
@Test |
||||
public void testRunSql() throws Exception { |
||||
FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.CLUSTER); |
||||
flinkParameters.setProgramType(ProgramType.SQL); |
||||
List<String> commandLine = FlinkArgsUtils.buildCommandLine(buildTestTaskExecutionContext(), flinkParameters); |
||||
|
||||
Assert.assertEquals("sql-client.sh -i /tmp/execution/app-id_init.sql -f /tmp/execution/app-id_node.sql", |
||||
joinStringListWithSpace(commandLine)); |
||||
} |
||||
|
||||
@Test |
||||
public void testInitOptionsInClusterMode() throws Exception { |
||||
List<String> initOptions = FlinkArgsUtils.buildInitOptionsForSql(buildTestFlinkParametersWithDeployMode(FlinkDeployMode.CLUSTER)); |
||||
Assert.assertEquals(2, initOptions.size()); |
||||
Assert.assertTrue(initOptions.contains("set execution.target=local")); |
||||
Assert.assertTrue(initOptions.contains("set parallelism.default=4")); |
||||
} |
||||
|
||||
@Test |
||||
public void testInitOptionsInApplicationMode() throws Exception { |
||||
List<String> initOptions = FlinkArgsUtils.buildInitOptionsForSql(buildTestFlinkParametersWithDeployMode(FlinkDeployMode.APPLICATION)); |
||||
Assert.assertEquals(6, initOptions.size()); |
||||
Assert.assertTrue(initOptions.contains("set execution.target=yarn-per-job")); |
||||
Assert.assertTrue(initOptions.contains("set taskmanager.numberOfTaskSlots=4")); |
||||
Assert.assertTrue(initOptions.contains("set yarn.application.name=demo-app-name")); |
||||
Assert.assertTrue(initOptions.contains("set jobmanager.memory.process.size=1024m")); |
||||
Assert.assertTrue(initOptions.contains("set taskmanager.memory.process.size=1024m")); |
||||
Assert.assertTrue(initOptions.contains("set parallelism.default=4")); |
||||
} |
||||
} |
||||
@ -1,116 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.flink; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; |
||||
import org.apache.dolphinscheduler.spi.utils.JSONUtils; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.api.mockito.PowerMockito; |
||||
import org.powermock.core.classloader.annotations.PowerMockIgnore; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
import java.util.Collections; |
||||
|
||||
import static org.powermock.api.mockito.PowerMockito.spy; |
||||
import static org.powermock.api.mockito.PowerMockito.when; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({ |
||||
JSONUtils.class |
||||
}) |
||||
@PowerMockIgnore({"javax.*"}) |
||||
public class FlinkTaskTest { |
||||
|
||||
@Test |
||||
public void testBuildCommand() { |
||||
String parameters = buildFlinkParameters(); |
||||
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class); |
||||
when(taskExecutionContext.getTaskParams()).thenReturn(parameters); |
||||
when(taskExecutionContext.getQueue()).thenReturn("default"); |
||||
FlinkTask flinkTask = spy(new FlinkTask(taskExecutionContext)); |
||||
flinkTask.init(); |
||||
Assert.assertEquals( |
||||
"flink run " + |
||||
"-m yarn-cluster " + |
||||
"-ys 1 " + |
||||
"-ynm TopSpeedWindowing " + |
||||
"-yjm 1G " + |
||||
"-ytm 1G " + |
||||
"-yqu default " + |
||||
"-p 2 -sae " + |
||||
"-c org.apache.flink.streaming.examples.windowing.TopSpeedWindowing " + |
||||
"TopSpeedWindowing.jar", flinkTask.buildCommand()); |
||||
} |
||||
|
||||
@Test |
||||
public void testBuildCommandWithFlinkSql() { |
||||
String parameters = buildFlinkParametersWithFlinkSql(); |
||||
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class); |
||||
when(taskExecutionContext.getTaskParams()).thenReturn(parameters); |
||||
when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); |
||||
when(taskExecutionContext.getTaskAppId()).thenReturn("4483"); |
||||
FlinkTask flinkTask = spy(new FlinkTask(taskExecutionContext)); |
||||
flinkTask.init(); |
||||
Assert.assertEquals("sql-client.sh -i /tmp/4483_init.sql -f /tmp/4483_node.sql", flinkTask.buildCommand()); |
||||
} |
||||
|
||||
private String buildFlinkParameters() { |
||||
ResourceInfo resource = new ResourceInfo(); |
||||
resource.setId(2); |
||||
resource.setResourceName("/TopSpeedWindowing.jar"); |
||||
resource.setRes("TopSpeedWindowing.jar"); |
||||
|
||||
FlinkParameters parameters = new FlinkParameters(); |
||||
parameters.setLocalParams(Collections.emptyList()); |
||||
parameters.setResourceList(Collections.emptyList()); |
||||
parameters.setProgramType(ProgramType.JAVA); |
||||
parameters.setMainClass("org.apache.flink.streaming.examples.windowing.TopSpeedWindowing"); |
||||
parameters.setMainJar(resource); |
||||
parameters.setDeployMode("cluster"); |
||||
parameters.setAppName("TopSpeedWindowing"); |
||||
parameters.setFlinkVersion(">=1.10"); |
||||
parameters.setJobManagerMemory("1G"); |
||||
parameters.setTaskManagerMemory("1G"); |
||||
parameters.setSlot(1); |
||||
parameters.setTaskManager(2); |
||||
parameters.setParallelism(2); |
||||
return JSONUtils.toJsonString(parameters); |
||||
} |
||||
|
||||
private String buildFlinkParametersWithFlinkSql() { |
||||
FlinkParameters parameters = new FlinkParameters(); |
||||
parameters.setLocalParams(Collections.emptyList()); |
||||
parameters.setInitScript("set sql-client.execution.result-mode=tableau;"); |
||||
parameters.setRawScript("selcet 11111;"); |
||||
parameters.setProgramType(ProgramType.SQL); |
||||
parameters.setMainClass(""); |
||||
parameters.setDeployMode("cluster"); |
||||
parameters.setAppName("FlinkSQL"); |
||||
parameters.setOthers(""); |
||||
parameters.setJobManagerMemory("1G"); |
||||
parameters.setTaskManagerMemory("1G"); |
||||
parameters.setParallelism(1); |
||||
parameters.setFlinkVersion(">=1.10"); |
||||
return JSONUtils.toJsonString(parameters); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue