Browse Source

fix work oom when task logs's size is very large (#11224)

3.1.0-release
microeastcowboy 2 years ago committed by GitHub
parent
commit
9a53c6ac34
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 41
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

41
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

@ -404,46 +404,29 @@ public abstract class AbstractCommandExecutor {
* @return app id list * @return app id list
*/ */
private List<String> getAppIds(String logPath) { private List<String> getAppIds(String logPath) {
List<String> logs = convertFile2List(logPath);
List<String> appIds = new ArrayList<>(); List<String> appIds = new ArrayList<>();
File file = new File(logPath);
if (!file.exists()) {
return appIds;
}
/* /*
* analysis log?get submited yarn application id * analysis log?get submited yarn application id
*/ */
for (String log : logs) { try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logPath), StandardCharsets.UTF_8))) {
String appId = findAppId(log); String line;
while ((line = br.readLine()) != null) {
String appId = findAppId(line);
if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) { if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) {
logger.info("find app id: {}", appId); logger.info("find app id: {}", appId);
appIds.add(appId); appIds.add(appId);
} }
} }
return appIds;
}
/**
* convert file to list
*
* @param filename file name
* @return line list
*/
private List<String> convertFile2List(String filename) {
List<String> lineList = new ArrayList<>(100);
File file = new File(filename);
if (!file.exists()) {
return lineList;
}
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), StandardCharsets.UTF_8))) {
String line;
while ((line = br.readLine()) != null) {
lineList.add(line);
}
} catch (Exception e) { } catch (Exception e) {
logger.error(String.format("read file: %s failed : ", filename), e); logger.error(String.format("read file: %s failed : ", logPath), e);
} }
return appIds;
return lineList;
} }
/** /**

Loading…
Cancel
Save