github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

fix #3900 kill multi yarn app in one job

pull/3/MERGE
Eights-LI 4 years ago
parent
4b502e361f
commit
8fdd6f547d
1 changed files with 343 additions and 328 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 671
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

671
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
Unescape View File

@ -18,8 +18,11 @@
package org.apache.dolphinscheduler.server.utils; package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
@ -38,368 +41,380 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
/** /**
* mainly used to get the start command line of a process. * mainly used to get the start command line of a process.
*/ */
public class ProcessUtils { public class ProcessUtils {
/** /**
* logger. * logger.
*/ */
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
/** /**
* Initialization regularization, solve the problem of pre-compilation performance, * Initialization regularization, solve the problem of pre-compilation performance,
* avoid the thread safety problem of multi-thread operation. * avoid the thread safety problem of multi-thread operation.
*/ */
private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)"); private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)");
private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)"); private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)");
/** /**
* build command line characters. * build command line characters.
* @param commandList command list *
* @return command * @param commandList command list
*/ * @return command
public static String buildCommandStr(List<String> commandList) { */
String cmdstr; public static String buildCommandStr(List<String> commandList) {
String[] cmd = commandList.toArray(new String[commandList.size()]); String cmdstr;
SecurityManager security = System.getSecurityManager(); String[] cmd = (String[]) commandList.toArray();
boolean allowAmbiguousCommands = false; SecurityManager security = System.getSecurityManager();
if (security == null) { boolean allowAmbiguousCommands = false;
allowAmbiguousCommands = true; if (security == null) {
String value = System.getProperty("jdk.lang.Process.allowAmbiguousCommands"); allowAmbiguousCommands = true;
if (value != null) { String value = System.getProperty("jdk.lang.Process.allowAmbiguousCommands");
allowAmbiguousCommands = !"false".equalsIgnoreCase(value); if (value != null) {
} allowAmbiguousCommands = !"false".equalsIgnoreCase(value);
} }
if (allowAmbiguousCommands) { }
if (allowAmbiguousCommands) {
String executablePath = new File(cmd[0]).getPath(); String executablePath = new File(cmd[0]).getPath();
if (needsEscaping(VERIFICATION_LEGACY, executablePath)) { if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
executablePath = quoteString(executablePath); executablePath = quoteString(executablePath);
} }
cmdstr = createCommandLine( cmdstr = createCommandLine(
VERIFICATION_LEGACY, executablePath, cmd); VERIFICATION_LEGACY, executablePath, cmd);
} else { } else {
String executablePath; String executablePath;
try { try {
executablePath = getExecutablePath(cmd[0]); executablePath = getExecutablePath(cmd[0]);
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
StringBuilder join = new StringBuilder(); StringBuilder join = new StringBuilder();
for (String s : cmd) { for (String s : cmd) {
join.append(s).append(' '); join.append(s).append(' ');
} }
cmd = getTokensFromCommand(join.toString()); cmd = getTokensFromCommand(join.toString());
executablePath = getExecutablePath(cmd[0]); executablePath = getExecutablePath(cmd[0]);
// Check new executable name once more // Check new executable name once more
if (security != null) { if (security != null) {
security.checkExec(executablePath); security.checkExec(executablePath);
} }
} }
cmdstr = createCommandLine( cmdstr = createCommandLine(
isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd); isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
}
return cmdstr;
}
/**
* get executable path.
*
* @param path path
* @return executable path
*/
private static String getExecutablePath(String path) {
boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
return fileToRun.getPath();
}
/**
* whether is shell file.
*
* @param executablePath executable path
* @return true if endsWith .CMD or .BAT
*/
private static boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
}
/**
* quote string.
*
* @param arg argument
* @return format arg
*/
private static String quoteString(String arg) {
StringBuilder argbuf = new StringBuilder(arg.length() + 2);
return argbuf.append('"').append(arg).append('"').toString();
}
/**
* get tokens from command.
*
* @param command command
* @return token string array
*/
private static String[] getTokensFromCommand(String command) {
ArrayList<String> matchList = new ArrayList<>(8);
Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
while (regexMatcher.find()) {
matchList.add(regexMatcher.group());
}
return matchList.toArray(new String[matchList.size()]);
}
/**
* Lazy Pattern.
*/
private static class LazyPattern {
// Escape-support version:
// "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
}
/**
* verification cmd bat.
*/
private static final int VERIFICATION_CMD_BAT = 0;
/**
* verification win32.
*/
private static final int VERIFICATION_WIN32 = 1;
/**
* verification legacy.
*/
private static final int VERIFICATION_LEGACY = 2;
/**
* escape verification.
*/
private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'}, {' ', '\t'}};
/**
* create command line.
* @param verificationType verification type
* @param executablePath executable path
* @param cmd cmd
* @return command line
*/
private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"').append(s);
if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
cmdbuf.append('\\');
} }
cmdbuf.append('"'); return cmdstr;
} else {
cmdbuf.append(s);
}
} }
return cmdbuf.toString();
} /**
* get executable path.
/** *
* whether is quoted. * @param path path
* @param noQuotesInside * @return executable path
* @param arg */
* @param errorMessage private static String getExecutablePath(String path) {
* @return boolean boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
*/
private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) { File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
int lastPos = arg.length() - 1; return fileToRun.getPath();
if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
// The argument has already been quoted.
if (noQuotesInside) {
if (arg.indexOf('"', 1) != lastPos) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
}
return true;
} }
if (noQuotesInside) {
if (arg.indexOf('"') >= 0) { /**
// There is ["] inside. * whether is shell file.
throw new IllegalArgumentException(errorMessage); *
} * @param executablePath executable path
* @return true if endsWith .CMD or .BAT
*/
private static boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
} }
return false;
} /**
* quote string.
/** *
* whether needs escaping. * @param arg argument
* * @return format arg
* @param verificationType verification type */
* @param arg arg private static String quoteString(String arg) {
* @return boolean return '"' + arg + '"';
*/
private static boolean needsEscaping(int verificationType, String arg) {
boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
if (!argIsQuoted) {
char[] testEscape = ESCAPE_VERIFICATION[verificationType];
for (int i = 0; i < testEscape.length; ++i) {
if (arg.indexOf(testEscape[i]) >= 0) {
return true;
}
}
} }
return false;
} /**
* get tokens from command.
/** *
* kill yarn application. * @param command command
* * @return token string array
* @param appIds app id list */
* @param logger logger private static String[] getTokensFromCommand(String command) {
* @param tenantCode tenant code ArrayList<String> matchList = new ArrayList<>(8);
* @param executePath execute path Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
*/ while (regexMatcher.find()) {
public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) { matchList.add(regexMatcher.group());
if (appIds.size() > 0) {
String appid = appIds.get(appIds.size() - 1);
String commandFile = String
.format("%s/%s.kill", executePath, appid);
String cmd = "yarn application -kill " + appid;
try {
StringBuilder sb = new StringBuilder();
sb.append("#!/bin/sh\n");
sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
sb.append("cd $BASEDIR\n");
if (CommonUtils.getSystemEnvPath() != null) {
sb.append("source " + CommonUtils.getSystemEnvPath() + "\n");
} }
sb.append("\n\n"); return (String[]) matchList.toArray();
sb.append(cmd); }
File f = new File(commandFile); /**
* Lazy Pattern.
*/
private static class LazyPattern {
// Escape-support version:
// "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
}
if (!f.exists()) { /**
FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); * verification cmd bat.
*/
private static final int VERIFICATION_CMD_BAT = 0;
/**
* verification win32.
*/
private static final int VERIFICATION_WIN32 = 1;
/**
* verification legacy.
*/
private static final int VERIFICATION_LEGACY = 2;
/**
* escape verification.
*/
private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'}, {' ', '\t'}};
/**
* create command line.
*
* @param verificationType verification type
* @param executablePath executable path
* @param cmd cmd
* @return command line
*/
private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"').append(s);
if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
cmdbuf.append('\\');
}
cmdbuf.append('"');
} else {
cmdbuf.append(s);
}
} }
return cmdbuf.toString();
}
String runCmd = "sh " + commandFile; /**
if (StringUtils.isNotEmpty(tenantCode)) { * whether is quoted.
runCmd = "sudo -u " + tenantCode + " " + runCmd; *
* @param noQuotesInside no quotes inside
* @param arg arg
* @param errorMessage error message
* @return boolean
*/
private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
int lastPos = arg.length() - 1;
if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
// The argument has already been quoted.
if (noQuotesInside) {
if (arg.indexOf('"', 1) != lastPos) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
}
return true;
}
if (noQuotesInside) {
if (arg.indexOf('"') >= 0) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
} }
return false;
}
logger.info("kill cmd:{}", runCmd); /**
* whether needs escaping.
*
* @param verificationType verification type
* @param arg arg
* @return boolean
*/
private static boolean needsEscaping(int verificationType, String arg) {
boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
if (!argIsQuoted) {
char[] testEscape = ESCAPE_VERIFICATION[verificationType];
for (char c : testEscape) {
if (arg.indexOf(c) >= 0) {
return true;
}
}
}
return false;
}
Runtime.getRuntime().exec(runCmd); /**
} catch (Exception e) { * kill yarn application.
logger.error("kill application error", e); *
} * @param appIds app id list
* @param logger logger
* @param tenantCode tenant code
* @param executePath execute path
*/
public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) {
if (CollectionUtils.isNotEmpty(appIds)) {
for (String appId : appIds) {
try {
ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
if (!applicationStatus.typeIsFinished()) {
String commandFile = String
.format("%s/%s.kill", executePath, appId);
String cmd = "yarn application -kill " + appId;
try {
StringBuilder sb = new StringBuilder();
sb.append("#!/bin/sh\n");
sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
sb.append("cd $BASEDIR\n");
if (CommonUtils.getSystemEnvPath() != null) {
sb.append("source ").append(CommonUtils.getSystemEnvPath()).append("\n");
}
sb.append("\n\n");
sb.append(cmd);
File f = new File(commandFile);
if (!f.exists()) {
FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
}
String runCmd = "sh " + commandFile;
if (StringUtils.isNotEmpty(tenantCode)) {
runCmd = "sudo -u " + tenantCode + " " + runCmd;
}
logger.info("kill cmd:{}", runCmd);
Runtime.getRuntime().exec(runCmd);
} catch (Exception e) {
logger.error(String.format("Kill yarn application app id [%s] failed: [%s]", appId, e.getMessage()));
}
}
} catch (Exception e) {
logger.error(String.format("Get yarn application app id [%s] status failed: [%s]", appId, e.getMessage()));
}
}
}
} }
}
/** /**
* kill tasks according to different task types. * kill tasks according to different task types.
* *
* @param taskExecutionContext taskExecutionContext * @param taskExecutionContext taskExecutionContext
*/ */
public static void kill(TaskExecutionContext taskExecutionContext) { public static void kill(TaskExecutionContext taskExecutionContext) {
try { try {
int processId = taskExecutionContext.getProcessId(); int processId = taskExecutionContext.getProcessId();
if (processId == 0) { if (processId == 0) {
logger.error("process kill failed, process id :{}, task id:{}", logger.error("process kill failed, process id :{}, task id:{}",
processId, taskExecutionContext.getTaskInstanceId()); processId, taskExecutionContext.getTaskInstanceId());
return; return;
} }
String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
logger.info("process id:{}, cmd:{}", processId, cmd); logger.info("process id:{}, cmd:{}", processId, cmd);
OSUtils.exeCmd(cmd); OSUtils.exeCmd(cmd);
// find log and kill yarn job // find log and kill yarn job
killYarnJob(taskExecutionContext); killYarnJob(taskExecutionContext);
} catch (Exception e) { } catch (Exception e) {
logger.error("kill task failed", e); logger.error("kill task failed", e);
} }
}
/**
* get pids str.
*
* @param processId process id
* @return pids
* @throws Exception exception
*/
public static String getPidsStr(int processId) throws Exception {
StringBuilder sb = new StringBuilder();
Matcher mat;
// pstree pid get sub pids
if (OSUtils.isMacOS()) {
String pids = OSUtils.exeCmd("pstree -sp " + processId);
mat = MACPATTERN.matcher(pids);
} else {
String pids = OSUtils.exeCmd("pstree -p " + processId);
mat = WINDOWSATTERN.matcher(pids);
} }
while (mat.find()) { /**
sb.append(mat.group(1)).append(" "); * get pids str.
} *
return sb.toString().trim(); * @param processId process id
} * @return pids
* @throws Exception exception
/** */
* find logs and kill yarn tasks. public static String getPidsStr(int processId) throws Exception {
* StringBuilder sb = new StringBuilder();
* @param taskExecutionContext taskExecutionContext Matcher mat;
*/ // pstree pid get sub pids
public static void killYarnJob(TaskExecutionContext taskExecutionContext) { if (OSUtils.isMacOS()) {
try { String pids = OSUtils.exeCmd("pstree -sp " + processId);
Thread.sleep(Constants.SLEEP_TIME_MILLIS); mat = MACPATTERN.matcher(pids);
LogClientService logClient = null; } else {
String log = null; String pids = OSUtils.exeCmd("pstree -p " + processId);
try { mat = WINDOWSATTERN.matcher(pids);
logClient = new LogClientService();
log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
Constants.RPC_PORT,
taskExecutionContext.getLogPath());
} finally {
if (logClient != null) {
logClient.close();
}
}
if (StringUtils.isNotEmpty(log)) {
List<String> appIds = LoggerUtils.getAppIds(log, logger);
String workerDir = taskExecutionContext.getExecutePath();
if (StringUtils.isEmpty(workerDir)) {
logger.error("task instance work dir is empty");
throw new RuntimeException("task instance work dir is empty");
} }
if (appIds.size() > 0) {
cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath()); while (mat.find()) {
sb.append(mat.group(1)).append(" ");
} }
} return sb.toString().trim();
}
} catch (Exception e) { /**
logger.error("kill yarn job failure",e); * find logs and kill yarn tasks.
*
* @param taskExecutionContext taskExecutionContext
*/
public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
LogClientService logClient = null;
String log;
try {
logClient = new LogClientService();
log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
Constants.RPC_PORT,
taskExecutionContext.getLogPath());
} finally {
if (logClient != null) {
logClient.close();
}
}
if (StringUtils.isNotEmpty(log)) {
List<String> appIds = LoggerUtils.getAppIds(log, logger);
String workerDir = taskExecutionContext.getExecutePath();
if (StringUtils.isEmpty(workerDir)) {
logger.error("task instance work dir is empty");
throw new RuntimeException("task instance work dir is empty");
}
if (appIds.size() > 0) {
cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
}
}
} catch (Exception e) {
logger.error("kill yarn job failure", e);
}
} }
}
} }

Write Preview
Loading…
Cancel
Save