[Improvement][Log] optimize task log path to reduce task running time (#6731)

* [Improvement][Log] optimize task log path to reduce task running time * Reduce the creation of log directories Co-authored-by: Kirs <acm_master@163.com>
3 years ago · f564687a89
12 changed files with 65 additions and 34 deletions
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@ -184,6 +184,10 @@ public final class Constants {
     */
    public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";

+    /**
+     * date format of yyyyMMdd
+     */
+    public static final String YYYYMMDD = "yyyyMMdd";

    /**
     * date format of yyyyMMddHHmmss
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
@ -92,7 +92,7 @@ public final class DateUtils {
    /**
     * get the formatted date string
     *
-     * @param date date
+     * @param date   date
     * @param format e.g. yyyy-MM-dd HH:mm:ss
     * @return date string
     */
@ -104,7 +104,7 @@ public final class DateUtils {
     * get the formatted date string
     *
     * @param localDateTime local data time
-     * @param format yyyy-MM-dd HH:mm:ss
+     * @param format        yyyy-MM-dd HH:mm:ss
     * @return date string
     */
    public static String format(LocalDateTime localDateTime, String format) {
@ -124,7 +124,7 @@ public final class DateUtils {
    /**
     * convert string to date and time
     *
-     * @param date date
+     * @param date   date
     * @param format format
     * @return date
     */
@ -177,7 +177,7 @@ public final class DateUtils {
     * get the date of the specified date in the days before and after
     *
     * @param date date
-     * @param day day
+     * @param day  day
     * @return the date of the specified date in the days before and after
     */
    public static Date getSomeDay(Date date, int day) {
@ -203,7 +203,7 @@ public final class DateUtils {
     * compare two dates
     *
     * @param future future date
-     * @param old old date
+     * @param old    old date
     * @return true if future time greater than old time
     */
    public static boolean compare(Date future, Date old) {
@ -329,7 +329,7 @@ public final class DateUtils {
    /**
     * get some hour of day
     *
-     * @param date date
+     * @param date       date
     * @param offsetHour hours
     * @return some hour of day
     */
@ -432,15 +432,15 @@ public final class DateUtils {
     */
    public static Date getCurrentDate() {
        return DateUtils.parse(DateUtils.getCurrentTime(),
-                Constants.YYYY_MM_DD_HH_MM_SS);
+            Constants.YYYY_MM_DD_HH_MM_SS);
    }

    /**
     * get date
     *
-     * @param date date
+     * @param date          date
     * @param calendarField calendarField
-     * @param amount amount
+     * @param amount        amount
     * @return date
     */
    public static Date add(final Date date, final int calendarField, final int amount) {
@ -457,7 +457,7 @@ public final class DateUtils {
     * starting from the current time, get how many seconds are left before the target time.
     * targetTime = baseTime + intervalSeconds
     *
-     * @param baseTime base time
+     * @param baseTime        base time
     * @param intervalSeconds a period of time
     * @return the number of seconds
     */
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
@ -24,6 +24,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@ -71,12 +72,14 @@ public class LoggerUtils {
     * @return task id format
     */
    public static String buildTaskId(String affix,
+                                     Date firstSubmitTime,
                                     Long processDefineCode,
                                     int processDefineVersion,
                                     int processInstId,
                                     int taskId) {
-        // - [taskAppId=TASK-798_1-4084-15210]
-        return String.format(" - %s%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, processDefineCode, processDefineVersion, processInstId, taskId);
+        // - [taskAppId=TASK-20211107-798_1-4084-15210]
+        String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, Constants.YYYYMMDD);
+        return String.format(" - %s%s-%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, firstSubmitTimeStr, processDefineCode, processDefineVersion, processInstId, taskId);
    }

    /**
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
@ -47,7 +47,7 @@ public class TaskLogDiscriminator extends AbstractDiscriminator<ILoggingEvent> {
        String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-";
        if (loggerName.startsWith(prefix)) {
            return loggerName.substring(prefix.length(),
-                    loggerName.length() - 1).replace("-","/");
+                    loggerName.length() - 1).replaceFirst("-","/");
        } else {
            return "unknown_task";
        }
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
@ -82,6 +82,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
        );

        logger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+                taskInstance.getFirstSubmitTime(),
                processInstance.getProcessDefinitionCode(),
                processInstance.getProcessDefinitionVersion(),
                taskInstance.getProcessInstanceId(),
@ -144,7 +145,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
    }

    private void initTaskParameters() {
-        taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
+        taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
                processInstance.getProcessDefinitionVersion(),
                taskInstance.getProcessInstanceId(),
                taskInstance.getId()));
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
@ -89,7 +89,8 @@ public class DependentTaskProcessor extends BaseTaskProcessor {
        taskDefinition = processService.findTaskDefinition(
                taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
        );
-        taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
+        taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),
+                processInstance.getProcessDefinitionCode(),
                processInstance.getProcessDefinitionVersion(),
                taskInstance.getProcessInstanceId(),
                taskInstance.getId()));
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
@ -71,7 +71,7 @@ public class SwitchTaskProcessor extends BaseTaskProcessor {
        taskDefinition = processService.findTaskDefinition(
                taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
        );
-        taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
+        taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
                processInstance.getProcessDefinitionVersion(),
                taskInstance.getProcessInstanceId(),
                taskInstance.getId()));
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java
@ -17,11 +17,14 @@

 package org.apache.dolphinscheduler.server.utils;

+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
 import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;

 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Date;
 import java.util.Optional;

 import org.slf4j.LoggerFactory;
@ -32,6 +35,8 @@ import ch.qos.logback.core.spi.AppenderAttachable;

 public class LogUtils {

+    public static final String LOG_TAILFIX = ".log";
+
    private LogUtils() throws IllegalStateException {
        throw new IllegalStateException("Utility class");
    }
@ -39,30 +44,39 @@ public class LogUtils {
    /**
     * get task log path
     */
-    public static String getTaskLogPath(Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) {
+    public static String getTaskLogPath(Date firstSubmitTime, Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) {
+        // format /logs/YYYYMMDD/defintion-code_defintion_version-processInstanceId-taskInstanceId.log
+        final String taskLogFileName = new StringBuilder(String.valueOf(processDefineCode))
+            .append(Constants.UNDERLINE)
+            .append(processDefineVersion)
+            .append(Constants.SUBTRACT_CHAR)
+            .append(processInstanceId)
+            .append(Constants.SUBTRACT_CHAR)
+            .append(taskInstanceId)
+            .append(LOG_TAILFIX)
+            .toString();
        // Optional.map will be skipped if null
        return Optional.of(LoggerFactory.getILoggerFactory())
-                .map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
-                .map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
-                .map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
-                .map(TaskLogDiscriminator::getLogBase)
-                .map(e -> Paths.get(e)
-                        .toAbsolutePath()
-                        .resolve(processDefineCode + "_" + processDefineVersion)
-                        .resolve(String.valueOf(processInstanceId))
-                        .resolve(taskInstanceId + ".log"))
-                .map(Path::toString)
-                .orElse("");
+            .map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
+            .map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
+            .map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
+            .map(TaskLogDiscriminator::getLogBase)
+            .map(e -> Paths.get(e)
+                .toAbsolutePath()
+                .resolve(DateUtils.format(firstSubmitTime,Constants.YYYYMMDD))
+                .resolve(taskLogFileName))
+            .map(Path::toString)
+            .orElse("");
    }

    /**
     * get task log path by TaskExecutionContext
     */
    public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) {
-        return getTaskLogPath(taskExecutionContext.getProcessDefineCode(),
-                taskExecutionContext.getProcessDefineVersion(),
-                taskExecutionContext.getProcessInstanceId(),
-                taskExecutionContext.getTaskInstanceId());
+        return getTaskLogPath(taskExecutionContext.getFirstSubmitTime(),taskExecutionContext.getProcessDefineCode(),
+            taskExecutionContext.getProcessDefineVersion(),
+            taskExecutionContext.getProcessInstanceId(),
+            taskExecutionContext.getTaskInstanceId());
    }

 }
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@ -169,6 +169,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
            }
            TaskRequest taskRequest = JSONUtils.parseObject(JSONUtils.toJsonString(taskExecutionContext), TaskRequest.class);
            String taskLogName = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+                    taskExecutionContext.getFirstSubmitTime(),
                    taskExecutionContext.getProcessDefineCode(),
                    taskExecutionContext.getProcessDefineVersion(),
                    taskExecutionContext.getProcessInstanceId(),
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
@ -121,7 +121,7 @@ public class TaskLogDiscriminatorTest {

            }
        });
-        Assert.assertEquals("1/1/", result);
+        Assert.assertEquals("1/1-", result);
    }

    @Test
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java
@ -17,11 +17,14 @@

 package org.apache.dolphinscheduler.server.utils;

+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
 import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;

 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Date;

 import org.junit.Assert;
 import org.junit.Test;
@ -38,11 +41,13 @@ public class LogUtilsTest {

    @Test
    public void testGetTaskLogPath() {
+        Date firstSubmitTime = new Date();
        TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
        taskExecutionContext.setProcessInstanceId(100);
        taskExecutionContext.setTaskInstanceId(1000);
        taskExecutionContext.setProcessDefineCode(1L);
        taskExecutionContext.setProcessDefineVersion(1);
+        taskExecutionContext.setFirstSubmitTime(firstSubmitTime);

        Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT");
        Assert.assertNotNull(rootLogger);
@ -60,7 +65,8 @@ public class LogUtilsTest {

        Path logPath = Paths.get(".").toAbsolutePath().getParent()
                .resolve(logBase)
-                .resolve("1_1").resolve("100").resolve("1000.log");
+                .resolve(DateUtils.format(firstSubmitTime, Constants.YYYYMMDD))
+                .resolve("1_1-100-1000.log");
        Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext));
    }

--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
@ -108,6 +108,7 @@ public class TaskExecuteProcessorTest {
                .thenReturn(workerConfig);

        Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+                taskExecutionContext.getFirstSubmitTime(),
                taskExecutionContext.getProcessDefineCode(),
                taskExecutionContext.getProcessDefineVersion(),
                taskExecutionContext.getProcessInstanceId(),