Browse Source
* Use MDC to collect task instance log * Use MDCAutoClosableContext to remove the MDC key3.2.0-release
Wenjun Ruan
2 years ago
committed by
GitHub
59 changed files with 506 additions and 1084 deletions
@ -0,0 +1,77 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<configuration scan="true" scanPeriod="120 seconds"> |
||||
<property name="log.base" value="logs"/> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<conversionRule conversionWord="message" |
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||
<filter class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter"/> |
||||
<Discriminator class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator"> |
||||
<key>taskInstanceLogFullPath</key> |
||||
<logBase>${log.base}</logBase> |
||||
</Discriminator> |
||||
<sift> |
||||
<appender name="FILE-${taskInstanceLogFullPath}" class="ch.qos.logback.core.FileAppender"> |
||||
<file>${taskInstanceLogFullPath}</file> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %message%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
<append>true</append> |
||||
</appender> |
||||
</sift> |
||||
</appender> |
||||
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-master.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
<totalSizeCap>50GB</totalSizeCap> |
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<if condition="${DOCKER:-false}"> |
||||
<then> |
||||
<appender-ref ref="STDOUT"/> |
||||
</then> |
||||
</if> |
||||
<appender-ref ref="TASKLOGFILE"/> |
||||
<appender-ref ref="MASTERLOGFILE"/> |
||||
</root> |
||||
</configuration> |
||||
@ -1,85 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.service.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
||||
|
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
import java.util.Date; |
||||
import java.util.Optional; |
||||
|
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import ch.qos.logback.classic.sift.SiftingAppender; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.core.spi.AppenderAttachable; |
||||
|
||||
public class LogUtils { |
||||
|
||||
public static final String LOG_TAILFIX = ".log"; |
||||
|
||||
private LogUtils() throws IllegalStateException { |
||||
throw new IllegalStateException("Utility class"); |
||||
} |
||||
|
||||
/** |
||||
* get task log path |
||||
*/ |
||||
public static String getTaskLogPath(Date firstSubmitTime, Long processDefineCode, int processDefineVersion, |
||||
int processInstanceId, int taskInstanceId) { |
||||
// format /logs/YYYYMMDD/defintion-code_defintion_version-processInstanceId-taskInstanceId.log
|
||||
final String taskLogFileName = new StringBuilder(String.valueOf(processDefineCode)) |
||||
.append(Constants.UNDERLINE) |
||||
.append(processDefineVersion) |
||||
.append(Constants.SUBTRACT_CHAR) |
||||
.append(processInstanceId) |
||||
.append(Constants.SUBTRACT_CHAR) |
||||
.append(taskInstanceId) |
||||
.append(LOG_TAILFIX) |
||||
.toString(); |
||||
// Optional.map will be skipped if null
|
||||
return Optional.of(LoggerFactory.getILoggerFactory()) |
||||
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT"))) |
||||
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE"))) |
||||
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator()))) |
||||
.map(TaskLogDiscriminator::getLogBase) |
||||
.map(e -> Paths.get(e) |
||||
.toAbsolutePath() |
||||
.resolve(DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null)) |
||||
.resolve(taskLogFileName)) |
||||
.map(Path::toString) |
||||
.orElse(""); |
||||
} |
||||
|
||||
/** |
||||
* get task log path by TaskExecutionContext |
||||
*/ |
||||
public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) { |
||||
return getTaskLogPath(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), |
||||
taskExecutionContext.getProcessDefineCode(), |
||||
taskExecutionContext.getProcessDefineVersion(), |
||||
taskExecutionContext.getProcessInstanceId(), |
||||
taskExecutionContext.getTaskInstanceId()); |
||||
} |
||||
|
||||
} |
||||
@ -1,119 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.service.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
|
||||
import java.io.BufferedReader; |
||||
import java.io.FileInputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStreamReader; |
||||
import java.util.Date; |
||||
|
||||
import lombok.experimental.UtilityClass; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.slf4j.MDC; |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils}. |
||||
*/ |
||||
@Deprecated |
||||
@UtilityClass |
||||
@Slf4j |
||||
public class LoggerUtils { |
||||
|
||||
public static String buildTaskId(Date firstSubmitTime, |
||||
Long processDefineCode, |
||||
int processDefineVersion, |
||||
int processInstId, |
||||
int taskId) { |
||||
// like TaskAppId=TASK-20211107-798_1-4084-15210
|
||||
String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null); |
||||
return String.format("%s=%s-%s-%s_%s-%s-%s", |
||||
TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr, |
||||
processDefineCode, processDefineVersion, processInstId, taskId); |
||||
} |
||||
|
||||
/** |
||||
* read whole file content |
||||
* |
||||
* @param filePath file path |
||||
* @return whole file content |
||||
*/ |
||||
public static String readWholeFileContent(String filePath) { |
||||
String line; |
||||
StringBuilder sb = new StringBuilder(); |
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath)))) { |
||||
while ((line = br.readLine()) != null) { |
||||
sb.append(line + "\r\n"); |
||||
} |
||||
return sb.toString(); |
||||
} catch (IOException e) { |
||||
log.error("read file error", e); |
||||
} |
||||
return ""; |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void setWorkflowAndTaskInstanceIDMDC(Integer workflowInstanceId, Integer taskInstanceId) { |
||||
setWorkflowInstanceIdMDC(workflowInstanceId); |
||||
setTaskInstanceIdMDC(taskInstanceId); |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void setWorkflowInstanceIdMDC(Integer workflowInstanceId) { |
||||
MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId)); |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void setTaskInstanceIdMDC(Integer taskInstanceId) { |
||||
MDC.put(Constants.TASK_INSTANCE_ID_MDC_KEY, String.valueOf(taskInstanceId)); |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void removeWorkflowAndTaskInstanceIdMDC() { |
||||
removeWorkflowInstanceIdMDC(); |
||||
removeTaskInstanceIdMDC(); |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void removeWorkflowInstanceIdMDC() { |
||||
MDC.remove(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY); |
||||
} |
||||
|
||||
/** |
||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
||||
*/ |
||||
public static void removeTaskInstanceIdMDC() { |
||||
MDC.remove(Constants.TASK_INSTANCE_ID_MDC_KEY); |
||||
} |
||||
} |
||||
@ -1,151 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.service.log; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; |
||||
import org.apache.dolphinscheduler.remote.utils.Host; |
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils; |
||||
|
||||
import java.nio.charset.StandardCharsets; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.MockedStatic; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class LogClientTest { |
||||
|
||||
@Test |
||||
public void testViewLogFromLocal() { |
||||
String localMachine = "LOCAL_MACHINE"; |
||||
int port = 1234; |
||||
String path = "/tmp/log"; |
||||
|
||||
try ( |
||||
MockedStatic<NetUtils> mockedNetUtils = Mockito.mockStatic(NetUtils.class); |
||||
MockedStatic<LoggerUtils> mockedLoggerUtils = Mockito.mockStatic(LoggerUtils.class)) { |
||||
mockedNetUtils.when(NetUtils::getHost) |
||||
.thenReturn(localMachine); |
||||
mockedLoggerUtils.when(() -> LoggerUtils.readWholeFileContent(Mockito.anyString())) |
||||
.thenReturn("application_xx_11"); |
||||
LogClient logClient = new LogClient(); |
||||
String log = logClient.viewLog(localMachine, port, path); |
||||
Assertions.assertNotNull(log); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testViewLogFromRemote() throws Exception { |
||||
String localMachine = "127.0.0.1"; |
||||
int port = 1234; |
||||
String path = "/tmp/log"; |
||||
|
||||
try (MockedStatic<NetUtils> mockedNetUtils = Mockito.mockStatic(NetUtils.class)) { |
||||
mockedNetUtils.when(NetUtils::getHost) |
||||
.thenReturn(localMachine + "1"); |
||||
LogClient logClient = new LogClient(); |
||||
String log = logClient.viewLog(localMachine, port, path); |
||||
Assertions.assertNotNull(log); |
||||
} |
||||
|
||||
Command command = new Command(); |
||||
command.setBody(JSONUtils.toJsonString(new ViewLogResponseCommand("")).getBytes(StandardCharsets.UTF_8)); |
||||
LogClient logClient = new LogClient(); |
||||
String log = logClient.viewLog(localMachine, port, path); |
||||
Assertions.assertNotNull(log); |
||||
} |
||||
|
||||
@Test |
||||
public void testClose() { |
||||
try ( |
||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
||||
.thenReturn(remotingClient); |
||||
LogClient logClient = new LogClient(); |
||||
logClient.close(); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testRollViewLog() throws Exception { |
||||
try ( |
||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
||||
.thenReturn(remotingClient); |
||||
Command command = new Command(); |
||||
command.setBody(JSONUtils.toJsonByteArray(new RollViewLogResponseCommand("success"))); |
||||
Mockito.when( |
||||
remotingClient.sendSync(Mockito.any(Host.class), Mockito.any(Command.class), Mockito.anyLong())) |
||||
.thenReturn(command); |
||||
|
||||
LogClient logClient = new LogClient(); |
||||
String msg = logClient.rollViewLog("localhost", 1234, "/tmp/log", 0, 10); |
||||
Assertions.assertNotNull(msg); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testGetLogBytes() throws Exception { |
||||
try ( |
||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
||||
.thenReturn(remotingClient); |
||||
Command command = new Command(); |
||||
command.setBody( |
||||
JSONUtils.toJsonByteArray(new GetLogBytesResponseCommand("log".getBytes(StandardCharsets.UTF_8)))); |
||||
Mockito.when( |
||||
remotingClient.sendSync(Mockito.any(Host.class), Mockito.any(Command.class), Mockito.anyLong())) |
||||
.thenReturn(command); |
||||
|
||||
LogClient logClient = new LogClient(); |
||||
byte[] logBytes = logClient.getLogBytes("localhost", 1234, "/tmp/log"); |
||||
Assertions.assertNotNull(logBytes); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testRemoveTaskLog() { |
||||
|
||||
try ( |
||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
||||
.thenReturn(remotingClient); |
||||
|
||||
LogClient logClient = new LogClient(); |
||||
Assertions.assertDoesNotThrow(() -> logClient.removeTaskLog(Host.of("localhost:1234"), "/log/path")); |
||||
} |
||||
} |
||||
} |
||||
@ -1,108 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.service.log; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.BeforeEach; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.LoggingEvent; |
||||
|
||||
public class TaskLogDiscriminatorTest { |
||||
|
||||
/** |
||||
* log base |
||||
*/ |
||||
private String logBase = "logs"; |
||||
|
||||
TaskLogDiscriminator taskLogDiscriminator; |
||||
|
||||
@BeforeEach |
||||
public void before() { |
||||
taskLogDiscriminator = new TaskLogDiscriminator(); |
||||
taskLogDiscriminator.setLogBase("logs"); |
||||
taskLogDiscriminator.setKey("123"); |
||||
} |
||||
|
||||
@Test |
||||
public void getDiscriminatingValue() { |
||||
String result = taskLogDiscriminator.getDiscriminatingValue(new LoggingEvent() { |
||||
|
||||
@Override |
||||
public String getThreadName() { |
||||
return "taskAppId=TASK-20220105-101-1-1001"; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return TaskConstants.TASK_LOG_LOGGER_NAME; |
||||
} |
||||
}); |
||||
Assertions.assertEquals("20220105/101-1-1001", result); |
||||
} |
||||
|
||||
@Test |
||||
public void start() { |
||||
taskLogDiscriminator.start(); |
||||
Assertions.assertEquals(true, taskLogDiscriminator.isStarted()); |
||||
} |
||||
|
||||
@Test |
||||
public void getKey() { |
||||
Assertions.assertEquals("123", taskLogDiscriminator.getKey()); |
||||
} |
||||
|
||||
@Test |
||||
public void setKey() { |
||||
|
||||
taskLogDiscriminator.setKey("123"); |
||||
} |
||||
|
||||
@Test |
||||
public void getLogBase() { |
||||
Assertions.assertEquals("logs", taskLogDiscriminator.getLogBase()); |
||||
} |
||||
|
||||
@Test |
||||
public void setLogBase() { |
||||
taskLogDiscriminator.setLogBase("logs"); |
||||
} |
||||
} |
||||
@ -1,71 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.service.log; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.LoggingEvent; |
||||
import ch.qos.logback.core.spi.FilterReply; |
||||
|
||||
public class TaskLogFilterTest { |
||||
|
||||
@Test |
||||
public void decide() { |
||||
TaskLogFilter taskLogFilter = new TaskLogFilter(); |
||||
|
||||
FilterReply filterReply = taskLogFilter.decide(new LoggingEvent() { |
||||
|
||||
@Override |
||||
public String getThreadName() { |
||||
return TaskConstants.TASK_APPID_LOG_FORMAT; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return Level.INFO; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return "raw script : echo 222"; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return "raw script : echo 222"; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return TaskConstants.TASK_LOG_LOGGER_NAME; |
||||
} |
||||
}); |
||||
|
||||
Assertions.assertEquals(FilterReply.ACCEPT, filterReply); |
||||
|
||||
} |
||||
} |
||||
@ -1,73 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.service.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
||||
|
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
import java.util.Date; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import ch.qos.logback.classic.Logger; |
||||
import ch.qos.logback.classic.sift.SiftingAppender; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class LogUtilsTest { |
||||
|
||||
@Test |
||||
public void testGetTaskLogPath() { |
||||
Date firstSubmitTime = new Date(); |
||||
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); |
||||
taskExecutionContext.setProcessInstanceId(100); |
||||
taskExecutionContext.setTaskInstanceId(1000); |
||||
taskExecutionContext.setProcessDefineCode(1L); |
||||
taskExecutionContext.setProcessDefineVersion(1); |
||||
taskExecutionContext.setFirstSubmitTime(firstSubmitTime.getTime()); |
||||
|
||||
Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT"); |
||||
Assertions.assertNotNull(rootLogger); |
||||
|
||||
SiftingAppender appender = Mockito.mock(SiftingAppender.class); |
||||
// it's a trick to mock logger.getAppend("TASKLOGFILE")
|
||||
Mockito.when(appender.getName()).thenReturn("TASKLOGFILE"); |
||||
rootLogger.addAppender(appender); |
||||
|
||||
Path logBase = Paths.get("path").resolve("to").resolve("test"); |
||||
|
||||
TaskLogDiscriminator taskLogDiscriminator = Mockito.mock(TaskLogDiscriminator.class); |
||||
Mockito.when(taskLogDiscriminator.getLogBase()).thenReturn(logBase.toString()); |
||||
Mockito.when(appender.getDiscriminator()).thenReturn(taskLogDiscriminator); |
||||
|
||||
Path logPath = Paths.get(".").toAbsolutePath().getParent() |
||||
.resolve(logBase) |
||||
.resolve(DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null)) |
||||
.resolve("1_1-100-1000.log"); |
||||
Assertions.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext)); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,78 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<configuration scan="true" scanPeriod="120 seconds"> |
||||
<property name="log.base" value="logs"/> |
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<conversionRule conversionWord="message" |
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||
<filter class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter"/> |
||||
<Discriminator class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator"> |
||||
<key>taskInstanceLogFullPath</key> |
||||
<logBase>${log.base}</logBase> |
||||
</Discriminator> |
||||
<sift> |
||||
<appender name="FILE-${taskInstanceLogFullPath}" class="ch.qos.logback.core.FileAppender"> |
||||
<file>${taskInstanceLogFullPath}</file> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %message%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
<append>true</append> |
||||
</appender> |
||||
</sift> |
||||
</appender> |
||||
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-worker.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
<totalSizeCap>50GB</totalSizeCap> |
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<if condition="${DOCKER:-false}"> |
||||
<then> |
||||
<appender-ref ref="STDOUT"/> |
||||
</then> |
||||
</if> |
||||
<appender-ref ref="TASKLOGFILE"/> |
||||
<appender-ref ref="WORKERLOGFILE"/> |
||||
</root> |
||||
</configuration> |
||||
Loading…
Reference in new issue