Browse Source

fix work oom when task logs's size is very large (#11224)

3.1.0-release
microeastcowboy 2 years ago committed by GitHub
parent
commit
9a53c6ac34
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 45
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

45
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

@ -404,48 +404,31 @@ public abstract class AbstractCommandExecutor {
* @return app id list * @return app id list
*/ */
private List<String> getAppIds(String logPath) { private List<String> getAppIds(String logPath) {
List<String> logs = convertFile2List(logPath);
List<String> appIds = new ArrayList<>(); List<String> appIds = new ArrayList<>();
/*
* analysis log?get submited yarn application id
*/
for (String log : logs) {
String appId = findAppId(log);
if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) {
logger.info("find app id: {}", appId);
appIds.add(appId);
}
}
return appIds;
}
/**
* convert file to list
*
* @param filename file name
* @return line list
*/
private List<String> convertFile2List(String filename) {
List<String> lineList = new ArrayList<>(100);
File file = new File(filename);
File file = new File(logPath);
if (!file.exists()) { if (!file.exists()) {
return lineList; return appIds;
} }
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), StandardCharsets.UTF_8))) { /*
* analysis log?get submited yarn application id
*/
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logPath), StandardCharsets.UTF_8))) {
String line; String line;
while ((line = br.readLine()) != null) { while ((line = br.readLine()) != null) {
lineList.add(line); String appId = findAppId(line);
if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) {
logger.info("find app id: {}", appId);
appIds.add(appId);
}
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(String.format("read file: %s failed : ", filename), e); logger.error(String.format("read file: %s failed : ", logPath), e);
} }
return appIds;
return lineList;
} }
/** /**
* find var pool * find var pool
* @param line * @param line

Loading…
Cancel
Save