Browse Source

[Improvement][Log] optimize task log path to reduce task running time (#6731)

* [Improvement][Log] optimize task log path to reduce task running time
* Reduce the creation of log directories


Co-authored-by: Kirs <acm_master@163.com>
3.0.0/version-upgrade
chouc 3 years ago committed by GitHub
parent
commit
f564687a89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  2. 20
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  3. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
  4. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
  5. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
  6. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
  7. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
  8. 46
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java
  9. 1
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
  10. 2
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
  11. 8
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java
  12. 1
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -184,6 +184,10 @@ public final class Constants {
*/ */
public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
/**
* date format of yyyyMMdd
*/
public static final String YYYYMMDD = "yyyyMMdd";
/** /**
* date format of yyyyMMddHHmmss * date format of yyyyMMddHHmmss

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@ -92,7 +92,7 @@ public final class DateUtils {
/** /**
* get the formatted date string * get the formatted date string
* *
* @param date date * @param date date
* @param format e.g. yyyy-MM-dd HH:mm:ss * @param format e.g. yyyy-MM-dd HH:mm:ss
* @return date string * @return date string
*/ */
@ -104,7 +104,7 @@ public final class DateUtils {
* get the formatted date string * get the formatted date string
* *
* @param localDateTime local data time * @param localDateTime local data time
* @param format yyyy-MM-dd HH:mm:ss * @param format yyyy-MM-dd HH:mm:ss
* @return date string * @return date string
*/ */
public static String format(LocalDateTime localDateTime, String format) { public static String format(LocalDateTime localDateTime, String format) {
@ -124,7 +124,7 @@ public final class DateUtils {
/** /**
* convert string to date and time * convert string to date and time
* *
* @param date date * @param date date
* @param format format * @param format format
* @return date * @return date
*/ */
@ -177,7 +177,7 @@ public final class DateUtils {
* get the date of the specified date in the days before and after * get the date of the specified date in the days before and after
* *
* @param date date * @param date date
* @param day day * @param day day
* @return the date of the specified date in the days before and after * @return the date of the specified date in the days before and after
*/ */
public static Date getSomeDay(Date date, int day) { public static Date getSomeDay(Date date, int day) {
@ -203,7 +203,7 @@ public final class DateUtils {
* compare two dates * compare two dates
* *
* @param future future date * @param future future date
* @param old old date * @param old old date
* @return true if future time greater than old time * @return true if future time greater than old time
*/ */
public static boolean compare(Date future, Date old) { public static boolean compare(Date future, Date old) {
@ -329,7 +329,7 @@ public final class DateUtils {
/** /**
* get some hour of day * get some hour of day
* *
* @param date date * @param date date
* @param offsetHour hours * @param offsetHour hours
* @return some hour of day * @return some hour of day
*/ */
@ -432,15 +432,15 @@ public final class DateUtils {
*/ */
public static Date getCurrentDate() { public static Date getCurrentDate() {
return DateUtils.parse(DateUtils.getCurrentTime(), return DateUtils.parse(DateUtils.getCurrentTime(),
Constants.YYYY_MM_DD_HH_MM_SS); Constants.YYYY_MM_DD_HH_MM_SS);
} }
/** /**
* get date * get date
* *
* @param date date * @param date date
* @param calendarField calendarField * @param calendarField calendarField
* @param amount amount * @param amount amount
* @return date * @return date
*/ */
public static Date add(final Date date, final int calendarField, final int amount) { public static Date add(final Date date, final int calendarField, final int amount) {
@ -457,7 +457,7 @@ public final class DateUtils {
* starting from the current time, get how many seconds are left before the target time. * starting from the current time, get how many seconds are left before the target time.
* targetTime = baseTime + intervalSeconds * targetTime = baseTime + intervalSeconds
* *
* @param baseTime base time * @param baseTime base time
* @param intervalSeconds a period of time * @param intervalSeconds a period of time
* @return the number of seconds * @return the number of seconds
*/ */

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -71,12 +72,14 @@ public class LoggerUtils {
* @return task id format * @return task id format
*/ */
public static String buildTaskId(String affix, public static String buildTaskId(String affix,
Date firstSubmitTime,
Long processDefineCode, Long processDefineCode,
int processDefineVersion, int processDefineVersion,
int processInstId, int processInstId,
int taskId) { int taskId) {
// - [taskAppId=TASK-798_1-4084-15210] // - [taskAppId=TASK-20211107-798_1-4084-15210]
return String.format(" - %s%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, processDefineCode, processDefineVersion, processInstId, taskId); String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, Constants.YYYYMMDD);
return String.format(" - %s%s-%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, firstSubmitTimeStr, processDefineCode, processDefineVersion, processInstId, taskId);
} }
/** /**

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java

@ -47,7 +47,7 @@ public class TaskLogDiscriminator extends AbstractDiscriminator<ILoggingEvent> {
String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-"; String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-";
if (loggerName.startsWith(prefix)) { if (loggerName.startsWith(prefix)) {
return loggerName.substring(prefix.length(), return loggerName.substring(prefix.length(),
loggerName.length() - 1).replace("-","/"); loggerName.length() - 1).replaceFirst("-","/");
} else { } else {
return "unknown_task"; return "unknown_task";
} }

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java

@ -82,6 +82,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
); );
logger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, logger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskInstance.getFirstSubmitTime(),
processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(), processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(), taskInstance.getProcessInstanceId(),
@ -144,7 +145,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
} }
private void initTaskParameters() { private void initTaskParameters() {
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(), taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(), processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(), taskInstance.getProcessInstanceId(),
taskInstance.getId())); taskInstance.getId()));

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java

@ -89,7 +89,8 @@ public class DependentTaskProcessor extends BaseTaskProcessor {
taskDefinition = processService.findTaskDefinition( taskDefinition = processService.findTaskDefinition(
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion() taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
); );
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(), taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(), processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(), taskInstance.getProcessInstanceId(),
taskInstance.getId())); taskInstance.getId()));

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java

@ -71,7 +71,7 @@ public class SwitchTaskProcessor extends BaseTaskProcessor {
taskDefinition = processService.findTaskDefinition( taskDefinition = processService.findTaskDefinition(
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion() taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
); );
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(), taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(), processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(), taskInstance.getProcessInstanceId(),
taskInstance.getId())); taskInstance.getId()));

46
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java

@ -17,11 +17,14 @@
package org.apache.dolphinscheduler.server.utils; package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext; import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Date;
import java.util.Optional; import java.util.Optional;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -32,6 +35,8 @@ import ch.qos.logback.core.spi.AppenderAttachable;
public class LogUtils { public class LogUtils {
public static final String LOG_TAILFIX = ".log";
private LogUtils() throws IllegalStateException { private LogUtils() throws IllegalStateException {
throw new IllegalStateException("Utility class"); throw new IllegalStateException("Utility class");
} }
@ -39,30 +44,39 @@ public class LogUtils {
/** /**
* get task log path * get task log path
*/ */
public static String getTaskLogPath(Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) { public static String getTaskLogPath(Date firstSubmitTime, Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) {
// format /logs/YYYYMMDD/defintion-code_defintion_version-processInstanceId-taskInstanceId.log
final String taskLogFileName = new StringBuilder(String.valueOf(processDefineCode))
.append(Constants.UNDERLINE)
.append(processDefineVersion)
.append(Constants.SUBTRACT_CHAR)
.append(processInstanceId)
.append(Constants.SUBTRACT_CHAR)
.append(taskInstanceId)
.append(LOG_TAILFIX)
.toString();
// Optional.map will be skipped if null // Optional.map will be skipped if null
return Optional.of(LoggerFactory.getILoggerFactory()) return Optional.of(LoggerFactory.getILoggerFactory())
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT"))) .map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE"))) .map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator()))) .map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
.map(TaskLogDiscriminator::getLogBase) .map(TaskLogDiscriminator::getLogBase)
.map(e -> Paths.get(e) .map(e -> Paths.get(e)
.toAbsolutePath() .toAbsolutePath()
.resolve(processDefineCode + "_" + processDefineVersion) .resolve(DateUtils.format(firstSubmitTime,Constants.YYYYMMDD))
.resolve(String.valueOf(processInstanceId)) .resolve(taskLogFileName))
.resolve(taskInstanceId + ".log")) .map(Path::toString)
.map(Path::toString) .orElse("");
.orElse("");
} }
/** /**
* get task log path by TaskExecutionContext * get task log path by TaskExecutionContext
*/ */
public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) { public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) {
return getTaskLogPath(taskExecutionContext.getProcessDefineCode(), return getTaskLogPath(taskExecutionContext.getFirstSubmitTime(),taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(), taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId()); taskExecutionContext.getTaskInstanceId());
} }
} }

1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java

@ -169,6 +169,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
} }
TaskRequest taskRequest = JSONUtils.parseObject(JSONUtils.toJsonString(taskExecutionContext), TaskRequest.class); TaskRequest taskRequest = JSONUtils.parseObject(JSONUtils.toJsonString(taskExecutionContext), TaskRequest.class);
String taskLogName = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, String taskLogName = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskExecutionContext.getFirstSubmitTime(),
taskExecutionContext.getProcessDefineCode(), taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(), taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getProcessInstanceId(),

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java

@ -121,7 +121,7 @@ public class TaskLogDiscriminatorTest {
} }
}); });
Assert.assertEquals("1/1/", result); Assert.assertEquals("1/1-", result);
} }
@Test @Test

8
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java

@ -17,11 +17,14 @@
package org.apache.dolphinscheduler.server.utils; package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext; import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Date;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -38,11 +41,13 @@ public class LogUtilsTest {
@Test @Test
public void testGetTaskLogPath() { public void testGetTaskLogPath() {
Date firstSubmitTime = new Date();
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
taskExecutionContext.setProcessInstanceId(100); taskExecutionContext.setProcessInstanceId(100);
taskExecutionContext.setTaskInstanceId(1000); taskExecutionContext.setTaskInstanceId(1000);
taskExecutionContext.setProcessDefineCode(1L); taskExecutionContext.setProcessDefineCode(1L);
taskExecutionContext.setProcessDefineVersion(1); taskExecutionContext.setProcessDefineVersion(1);
taskExecutionContext.setFirstSubmitTime(firstSubmitTime);
Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT"); Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT");
Assert.assertNotNull(rootLogger); Assert.assertNotNull(rootLogger);
@ -60,7 +65,8 @@ public class LogUtilsTest {
Path logPath = Paths.get(".").toAbsolutePath().getParent() Path logPath = Paths.get(".").toAbsolutePath().getParent()
.resolve(logBase) .resolve(logBase)
.resolve("1_1").resolve("100").resolve("1000.log"); .resolve(DateUtils.format(firstSubmitTime, Constants.YYYYMMDD))
.resolve("1_1-100-1000.log");
Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext)); Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext));
} }

1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java

@ -108,6 +108,7 @@ public class TaskExecuteProcessorTest {
.thenReturn(workerConfig); .thenReturn(workerConfig);
Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskExecutionContext.getFirstSubmitTime(),
taskExecutionContext.getProcessDefineCode(), taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(), taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getProcessInstanceId(),

Loading…
Cancel
Save