github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improvement-14371][Hivecli] Add support for hivevar setting in hivecli task (#14423)

3.2.1-prepare
Rick Cheng 2 years ago committed by GitHub
parent
dec066f8ce
commit
13dfa859b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 11 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 71
      dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
  2. 25
      dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java

71
dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
Unescape View File

@ -18,8 +18,10 @@
package org.apache.dolphinscheduler.plugin.task.hivecli;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
@ -31,12 +33,23 @@ import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class HiveCliTask extends AbstractRemoteTask {
@ -108,29 +121,41 @@ public class HiveCliTask extends AbstractRemoteTask {
final String type = hiveCliParameters.getHiveCliTaskExecutionType();
String sqlContent = "";
String resourceFileName = "";
// TODO: make sure type is not unknown
if (HiveCliConstants.TYPE_FILE.equals(type)) {
args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
final List<ResourceInfo> resourceInfos = hiveCliParameters.getResourceList();
if (resourceInfos.size() > 1) {
log.warn("more than 1 files detected, use the first one by default");
}
args.add(StringUtils.stripStart(resourceInfos.get(0).getResourceName(), "/"));
try {
resourceFileName = resourceInfos.get(0).getResourceName();
sqlContent = FileUtils.readFileToString(
new File(String.format("%s/%s", taskExecutionContext.getExecutePath(), resourceFileName)),
StandardCharsets.UTF_8);
} catch (IOException e) {
log.error("read hive sql content from file {} error ", resourceFileName, e);
throw new TaskException("read hive sql content error", e);
}
} else {
final String script = hiveCliParameters.getHiveSqlScript();
args.add(String.format(HiveCliConstants.HIVE_CLI_EXECUTE_SCRIPT, script));
sqlContent = hiveCliParameters.getHiveSqlScript();
}
final Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
sqlContent = ParameterUtils.convertParameterPlaceholders(sqlContent, ParameterUtils.convert(paramsMap));
log.info("HiveCli sql content: {}", sqlContent);
String sqlFilePath = generateSqlScriptFile(sqlContent);
args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
args.add(new File(sqlFilePath).getName());
final String hiveCliOptions = hiveCliParameters.getHiveCliOptions();
if (StringUtils.isNotEmpty(hiveCliOptions)) {
args.add(hiveCliOptions);
}
final Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
final String command =
ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParameterUtils.convert(paramsMap));
String command = String.join(" ", args);
log.info("hiveCli task command: {}", command);
return command;
@ -151,4 +176,34 @@ public class HiveCliTask extends AbstractRemoteTask {
}
}
protected String generateSqlScriptFile(String rawScript) {
String scriptFileName = String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(),
taskExecutionContext.getTaskAppId());
File file = new File(scriptFileName);
Path path = file.toPath();
if (!Files.exists(path)) {
String script = rawScript.replaceAll("\\r\\n", "\n");
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(RWXR_XR_X);
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
try {
if (OSUtils.isWindows()) {
Files.createFile(path);
} else {
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
Files.createFile(path, attr);
}
Files.write(path, script.getBytes(), StandardOpenOption.APPEND);
} catch (IOException e) {
log.error("generate hivecli sql script error", e);
throw new TaskException("generate hivecli sql script error", e);
}
}
return scriptFileName;
}
}

25
dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
Unescape View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.hivecli;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@ -24,12 +25,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.commons.io.FileUtils;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ -37,13 +43,25 @@ import org.mockito.junit.jupiter.MockitoExtension;
public class HiveCliTaskTest {
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND =
"hive -e \"SHOW DATABASES;\"";
"hive -f 123_node.sql";
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND =
"hive -f sql_tasks/hive_task.sql";
"hive -f 123_node.sql";
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS =
"hive -e \"SHOW DATABASES;\" --verbose";
"hive -f 123_node.sql --verbose";
private MockedStatic<FileUtils> mockedStaticFileUtils;
@BeforeEach
public void setUp() {
mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class);
}
@AfterEach
public void after() {
mockedStaticFileUtils.close();
}
@Test
public void hiveCliTaskExecuteSqlFromScript() throws Exception {
@ -73,6 +91,7 @@ public class HiveCliTaskTest {
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(hiveCliTaskParameters);
HiveCliTask hiveCliTask = spy(new HiveCliTask(taskExecutionContext));
doReturn("123_node.sql").when(hiveCliTask).generateSqlScriptFile(Mockito.any());
return hiveCliTask;
}

Write Preview
Loading…
Cancel
Save