From e2cf4c50b9ff5d4216364c0a1761df790864f8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=94=E5=8D=A0=E5=8D=AB?= <825193156@qq.com> Date: Tue, 3 Sep 2019 17:42:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?Tgt=20invalid=20#742=20fix=EF=BC=8CConcurre?= =?UTF-8?q?nt=20task=20log=20bug=20#730=20fix=EF=BC=8Clogback.xml=20set=20?= =?UTF-8?q?log=20path=20support=20#723=20fix=20(#761)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mission log disorder，bug #751 fix * the log path of the task and the log path of the task instance are different. The log cannot be viewed. #723 * the log path of the task and the log path of the task instance are different. The log cannot be viewed. #723 bug fix * after starting kerberos authentication, tgt expires after one day，bug #742 fix * log pattern modify * LoggerServer remove comment code and ShellCommandExecutor modify * PythonCommandExecutor modify * Concurrent task log bug #730 fix * remove invalid commit --- .../escheduler/common/utils/HadoopUtils.java | 4 +++ .../escheduler/server/rpc/LoggerServer.java | 33 ++----------------- .../escheduler/server/utils/LoggerUtils.java | 7 ++-- .../worker/log/TaskLogDiscriminator.java | 21 ++++++++++-- .../server/worker/log/TaskLogFilter.java | 2 +- .../worker/runner/TaskScheduleThread.java | 17 ++++++++-- .../worker/task/AbstractCommandExecutor.java | 9 ++--- .../worker/task/PythonCommandExecutor.java | 5 --- .../worker/task/ShellCommandExecutor.java | 5 --- .../src/main/resources/worker_logback.xml | 1 + 10 files changed, 49 insertions(+), 55 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index 1ff41c37f7..6bc33eca42 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -66,6 +66,10 @@ public class HadoopUtils implements Closeable { } public static HadoopUtils getInstance(){ + // if kerberos startup , renew HadoopUtils + if (CommonUtils.getKerberosStartupState()){ + return new HadoopUtils(); + } return instance; } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/rpc/LoggerServer.java b/escheduler-server/src/main/java/cn/escheduler/server/rpc/LoggerServer.java index 705596c334..1dde113545 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/rpc/LoggerServer.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/rpc/LoggerServer.java @@ -100,13 +100,7 @@ public class LoggerServer { StringBuilder sb = new StringBuilder(); boolean errorLineFlag = false; for (String line : list){ - if (line.contains("TaskLogger")){ - errorLineFlag = filterLine(request.getPath(),line); - } - - if (!errorLineFlag || !line.contains("TaskLogger")){ - sb.append(line + "\r\n"); - } + sb.append(line + "\r\n"); } RetStrInfo retInfoBuild = RetStrInfo.newBuilder().setMsg(sb.toString()).build(); responseObserver.onNext(retInfoBuild); @@ -204,13 +198,7 @@ public class LoggerServer { br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); boolean errorLineFlag = false; while ((line = br.readLine()) != null){ - if (line.contains("TaskLogger")){ - errorLineFlag = filterLine(path,line); - } - - if (!errorLineFlag || !line.contains("TaskLogger")){ - sb.append(line + "\r\n"); - } + sb.append(line + "\r\n"); } return sb.toString(); @@ -228,21 +216,4 @@ public class LoggerServer { return null; } - - /** - * - * @param path - * @param line - * @return - */ - private static boolean filterLine(String path,String line){ - String removeSuffix = path.substring(0, path.length() - 4); - String[] strArrs = removeSuffix.split("/"); - String taskAppId = String.format("%s_%s_%s", - strArrs[strArrs.length - 3], - strArrs[strArrs.length-2], - strArrs[strArrs.length - 1]); - return !line.contains(taskAppId); - } - } \ No newline at end of file diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java index ab141e8629..a4207d3793 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java @@ -37,7 +37,9 @@ public class LoggerUtils { /** * Task Logger's prefix */ - public static final String TASK_LOGGER_INFO_PREFIX = "TaskLogInfo"; + public static final String TASK_LOGGER_INFO_PREFIX = "TASK"; + + public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo"; /** * build job id @@ -51,7 +53,8 @@ public class LoggerUtils { int processDefId, int processInstId, int taskId){ - return String.format("%s-%s/%s/%s",affix, + // - [taskAppId=TASK_79_4084_15210] + return String.format(" - [taskAppId=%s-%s-%s-%s]",affix, processDefId, processInstId, taskId); diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogDiscriminator.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogDiscriminator.java index 172b1b270b..91852725d3 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogDiscriminator.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogDiscriminator.java @@ -18,21 +18,27 @@ package cn.escheduler.server.worker.log; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.sift.AbstractDiscriminator; +import cn.escheduler.common.Constants; import cn.escheduler.server.utils.LoggerUtils; public class TaskLogDiscriminator extends AbstractDiscriminator { private String key; + private String logBase; + /** * logger name should be like: - * Task Logger name should be like: TaskLogInfo-{processDefinitionId}/{processInstanceId}/{taskInstanceId} + * Task Logger name should be like: Task-{processDefinitionId}-{processInstanceId}-{taskInstanceId} */ + @Override public String getDiscriminatingValue(ILoggingEvent event) { - String loggerName = event.getLoggerName(); + String loggerName = event.getLoggerName() + .split(Constants.EQUAL_SIGN)[1]; String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-"; if (loggerName.startsWith(prefix)) { - return loggerName.substring(prefix.length()); + return loggerName.substring(prefix.length(), + loggerName.length() - 1).replace("-","/"); } else { return "unknown_task"; } @@ -43,6 +49,7 @@ public class TaskLogDiscriminator extends AbstractDiscriminator { started = true; } + @Override public String getKey() { return key; } @@ -50,4 +57,12 @@ public class TaskLogDiscriminator extends AbstractDiscriminator { public void setKey(String key) { this.key = key; } + + public String getLogBase() { + return logBase; + } + + public void setLogBase(String logBase) { + this.logBase = logBase; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogFilter.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogFilter.java index 26ef5e84a8..bb93f52cd5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogFilter.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/log/TaskLogFilter.java @@ -28,7 +28,7 @@ public class TaskLogFilter extends Filter { @Override public FilterReply decide(ILoggingEvent event) { - if (event.getLoggerName().startsWith(LoggerUtils.TASK_LOGGER_INFO_PREFIX)) { + if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME)) { return FilterReply.ACCEPT; } return FilterReply.DENY; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index a839e3f706..542d741f42 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -17,6 +17,8 @@ package cn.escheduler.server.worker.runner; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.sift.SiftingAppender; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; import cn.escheduler.common.enums.TaskRecordStatus; @@ -40,6 +42,7 @@ import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.dao.model.Tenant; import cn.escheduler.server.utils.LoggerUtils; import cn.escheduler.server.utils.ParamUtils; +import cn.escheduler.server.worker.log.TaskLogDiscriminator; import cn.escheduler.server.worker.task.AbstractTask; import cn.escheduler.server.worker.task.TaskManager; import cn.escheduler.server.worker.task.TaskProps; @@ -109,7 +112,7 @@ public class TaskScheduleThread implements Runnable { // get tenant info Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), - processDefine.getUserId()); + processDefine.getUserId()); if(tenant == null){ logger.error("cannot find the tenant, process definition id:{}, user id:{}", @@ -220,8 +223,18 @@ public class TaskScheduleThread implements Runnable { * @return */ private String getTaskLogPath() { + String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory()) + .getLogger("ROOT") + .getAppender("TASKLOGFILE")) + .getDiscriminator()).getLogBase(); + if (baseLog.startsWith(Constants.SINGLE_SLASH)){ + return baseLog + Constants.SINGLE_SLASH + + taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH + + taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH + + taskInstance.getId() + ".log"; + } return System.getProperty("user.dir") + Constants.SINGLE_SLASH + - "logs" + Constants.SINGLE_SLASH + + baseLog + Constants.SINGLE_SLASH + taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH + taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH + taskInstance.getId() + ".log"; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java index 9a9123a415..e5ea6b725b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java @@ -22,6 +22,7 @@ import cn.escheduler.common.thread.ThreadUtils; import cn.escheduler.common.utils.HadoopUtils; import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.model.TaskInstance; +import cn.escheduler.server.utils.LoggerUtils; import cn.escheduler.server.utils.ProcessUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -347,7 +348,7 @@ public abstract class AbstractCommandExecutor { * get the standard output of the process */ private void parseProcessOutput(Process process) { - String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskAppId); + String threadLoggerInfoName = String.format(LoggerUtils.TASK_LOGGER_THREAD_NAME + "-%s", taskAppId); ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName); parseProcessOutputExecutorService.submit(new Runnable(){ @Override @@ -361,10 +362,7 @@ public abstract class AbstractCommandExecutor { long lastFlushTime = System.currentTimeMillis(); while ((line = inReader.readLine()) != null) { - if(checkShowLog(line)){ - logBuffer.add(line); - } - + logBuffer.add(line); lastFlushTime = flush(lastFlushTime); } } catch (Exception e) { @@ -566,7 +564,6 @@ public abstract class AbstractCommandExecutor { protected abstract String buildCommandFilePath(); protected abstract String commandType(); - protected abstract boolean checkShowLog(String line); protected abstract boolean checkFindApp(String line); protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java index 09a6add48f..6ea70f96f5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java @@ -100,11 +100,6 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { return pythonHome; } - @Override - protected boolean checkShowLog(String line) { - return true; - } - @Override protected boolean checkFindApp(String line) { return true; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/ShellCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/ShellCommandExecutor.java index 68e36b704a..d826fee9ca 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/ShellCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/ShellCommandExecutor.java @@ -60,11 +60,6 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { return SH; } - @Override - protected boolean checkShowLog(String line) { - return line.contains(taskAppId) || !line.contains("cn.escheduler.server.worker.log.TaskLogger"); - } - @Override protected boolean checkFindApp(String line) { return line.contains(taskAppId); diff --git a/escheduler-server/src/main/resources/worker_logback.xml b/escheduler-server/src/main/resources/worker_logback.xml index f630559da9..b3926f1a78 100644 --- a/escheduler-server/src/main/resources/worker_logback.xml +++ b/escheduler-server/src/main/resources/worker_logback.xml @@ -16,6 +16,7 @@ taskAppId + ${log.base} From a420a012a09129a520f2203f58599317c6e0ece2 Mon Sep 17 00:00:00 2001 From: Baoqi Date: Wed, 4 Sep 2019 10:44:53 +0800 Subject: [PATCH 2/2] also apply log path support #723 for combined server (#762) --- escheduler-api/src/main/resources/combined_logback.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/resources/combined_logback.xml b/escheduler-api/src/main/resources/combined_logback.xml index 6a186b09b7..fa1a95eeb2 100644 --- a/escheduler-api/src/main/resources/combined_logback.xml +++ b/escheduler-api/src/main/resources/combined_logback.xml @@ -16,6 +16,7 @@ taskAppId + ${log.base}