Browse Source
* Use MDC to collect task instance log * Use MDCAutoClosableContext to remove the MDC key3.2.0-release
Wenjun Ruan
2 years ago
committed by
GitHub
59 changed files with 506 additions and 1084 deletions
@ -0,0 +1,77 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> |
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<conversionRule conversionWord="message" |
||||||
|
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||||
|
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||||
|
<filter class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter"/> |
||||||
|
<Discriminator class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator"> |
||||||
|
<key>taskInstanceLogFullPath</key> |
||||||
|
<logBase>${log.base}</logBase> |
||||||
|
</Discriminator> |
||||||
|
<sift> |
||||||
|
<appender name="FILE-${taskInstanceLogFullPath}" class="ch.qos.logback.core.FileAppender"> |
||||||
|
<file>${taskInstanceLogFullPath}</file> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %message%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
<append>true</append> |
||||||
|
</appender> |
||||||
|
</sift> |
||||||
|
</appender> |
||||||
|
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-master.log</file> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>200MB</maxFileSize> |
||||||
|
<totalSizeCap>50GB</totalSizeCap> |
||||||
|
<cleanHistoryOnStart>true</cleanHistoryOnStart> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<if condition="${DOCKER:-false}"> |
||||||
|
<then> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
<appender-ref ref="TASKLOGFILE"/> |
||||||
|
<appender-ref ref="MASTERLOGFILE"/> |
||||||
|
</root> |
||||||
|
</configuration> |
||||||
@ -1,85 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.utils; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
|
||||||
|
|
||||||
import java.nio.file.Path; |
|
||||||
import java.nio.file.Paths; |
|
||||||
import java.util.Date; |
|
||||||
import java.util.Optional; |
|
||||||
|
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.sift.SiftingAppender; |
|
||||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
|
||||||
import ch.qos.logback.core.spi.AppenderAttachable; |
|
||||||
|
|
||||||
public class LogUtils { |
|
||||||
|
|
||||||
public static final String LOG_TAILFIX = ".log"; |
|
||||||
|
|
||||||
private LogUtils() throws IllegalStateException { |
|
||||||
throw new IllegalStateException("Utility class"); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get task log path |
|
||||||
*/ |
|
||||||
public static String getTaskLogPath(Date firstSubmitTime, Long processDefineCode, int processDefineVersion, |
|
||||||
int processInstanceId, int taskInstanceId) { |
|
||||||
// format /logs/YYYYMMDD/defintion-code_defintion_version-processInstanceId-taskInstanceId.log
|
|
||||||
final String taskLogFileName = new StringBuilder(String.valueOf(processDefineCode)) |
|
||||||
.append(Constants.UNDERLINE) |
|
||||||
.append(processDefineVersion) |
|
||||||
.append(Constants.SUBTRACT_CHAR) |
|
||||||
.append(processInstanceId) |
|
||||||
.append(Constants.SUBTRACT_CHAR) |
|
||||||
.append(taskInstanceId) |
|
||||||
.append(LOG_TAILFIX) |
|
||||||
.toString(); |
|
||||||
// Optional.map will be skipped if null
|
|
||||||
return Optional.of(LoggerFactory.getILoggerFactory()) |
|
||||||
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT"))) |
|
||||||
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE"))) |
|
||||||
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator()))) |
|
||||||
.map(TaskLogDiscriminator::getLogBase) |
|
||||||
.map(e -> Paths.get(e) |
|
||||||
.toAbsolutePath() |
|
||||||
.resolve(DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null)) |
|
||||||
.resolve(taskLogFileName)) |
|
||||||
.map(Path::toString) |
|
||||||
.orElse(""); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get task log path by TaskExecutionContext |
|
||||||
*/ |
|
||||||
public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) { |
|
||||||
return getTaskLogPath(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), |
|
||||||
taskExecutionContext.getProcessDefineCode(), |
|
||||||
taskExecutionContext.getProcessDefineVersion(), |
|
||||||
taskExecutionContext.getProcessInstanceId(), |
|
||||||
taskExecutionContext.getTaskInstanceId()); |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -1,119 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.utils; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
|
||||||
|
|
||||||
import java.io.BufferedReader; |
|
||||||
import java.io.FileInputStream; |
|
||||||
import java.io.IOException; |
|
||||||
import java.io.InputStreamReader; |
|
||||||
import java.util.Date; |
|
||||||
|
|
||||||
import lombok.experimental.UtilityClass; |
|
||||||
import lombok.extern.slf4j.Slf4j; |
|
||||||
|
|
||||||
import org.slf4j.MDC; |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils}. |
|
||||||
*/ |
|
||||||
@Deprecated |
|
||||||
@UtilityClass |
|
||||||
@Slf4j |
|
||||||
public class LoggerUtils { |
|
||||||
|
|
||||||
public static String buildTaskId(Date firstSubmitTime, |
|
||||||
Long processDefineCode, |
|
||||||
int processDefineVersion, |
|
||||||
int processInstId, |
|
||||||
int taskId) { |
|
||||||
// like TaskAppId=TASK-20211107-798_1-4084-15210
|
|
||||||
String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null); |
|
||||||
return String.format("%s=%s-%s-%s_%s-%s-%s", |
|
||||||
TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr, |
|
||||||
processDefineCode, processDefineVersion, processInstId, taskId); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* read whole file content |
|
||||||
* |
|
||||||
* @param filePath file path |
|
||||||
* @return whole file content |
|
||||||
*/ |
|
||||||
public static String readWholeFileContent(String filePath) { |
|
||||||
String line; |
|
||||||
StringBuilder sb = new StringBuilder(); |
|
||||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath)))) { |
|
||||||
while ((line = br.readLine()) != null) { |
|
||||||
sb.append(line + "\r\n"); |
|
||||||
} |
|
||||||
return sb.toString(); |
|
||||||
} catch (IOException e) { |
|
||||||
log.error("read file error", e); |
|
||||||
} |
|
||||||
return ""; |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void setWorkflowAndTaskInstanceIDMDC(Integer workflowInstanceId, Integer taskInstanceId) { |
|
||||||
setWorkflowInstanceIdMDC(workflowInstanceId); |
|
||||||
setTaskInstanceIdMDC(taskInstanceId); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void setWorkflowInstanceIdMDC(Integer workflowInstanceId) { |
|
||||||
MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId)); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void setTaskInstanceIdMDC(Integer taskInstanceId) { |
|
||||||
MDC.put(Constants.TASK_INSTANCE_ID_MDC_KEY, String.valueOf(taskInstanceId)); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void removeWorkflowAndTaskInstanceIdMDC() { |
|
||||||
removeWorkflowInstanceIdMDC(); |
|
||||||
removeTaskInstanceIdMDC(); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void removeWorkflowInstanceIdMDC() { |
|
||||||
MDC.remove(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Please use {@link org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils} |
|
||||||
*/ |
|
||||||
public static void removeTaskInstanceIdMDC() { |
|
||||||
MDC.remove(Constants.TASK_INSTANCE_ID_MDC_KEY); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,151 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.log; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
|
||||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
|
||||||
import org.apache.dolphinscheduler.remote.command.Command; |
|
||||||
import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand; |
|
||||||
import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; |
|
||||||
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; |
|
||||||
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; |
|
||||||
import org.apache.dolphinscheduler.remote.utils.Host; |
|
||||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils; |
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
import org.junit.jupiter.api.extension.ExtendWith; |
|
||||||
import org.mockito.MockedStatic; |
|
||||||
import org.mockito.Mockito; |
|
||||||
import org.mockito.junit.jupiter.MockitoExtension; |
|
||||||
|
|
||||||
@ExtendWith(MockitoExtension.class) |
|
||||||
public class LogClientTest { |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testViewLogFromLocal() { |
|
||||||
String localMachine = "LOCAL_MACHINE"; |
|
||||||
int port = 1234; |
|
||||||
String path = "/tmp/log"; |
|
||||||
|
|
||||||
try ( |
|
||||||
MockedStatic<NetUtils> mockedNetUtils = Mockito.mockStatic(NetUtils.class); |
|
||||||
MockedStatic<LoggerUtils> mockedLoggerUtils = Mockito.mockStatic(LoggerUtils.class)) { |
|
||||||
mockedNetUtils.when(NetUtils::getHost) |
|
||||||
.thenReturn(localMachine); |
|
||||||
mockedLoggerUtils.when(() -> LoggerUtils.readWholeFileContent(Mockito.anyString())) |
|
||||||
.thenReturn("application_xx_11"); |
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
String log = logClient.viewLog(localMachine, port, path); |
|
||||||
Assertions.assertNotNull(log); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testViewLogFromRemote() throws Exception { |
|
||||||
String localMachine = "127.0.0.1"; |
|
||||||
int port = 1234; |
|
||||||
String path = "/tmp/log"; |
|
||||||
|
|
||||||
try (MockedStatic<NetUtils> mockedNetUtils = Mockito.mockStatic(NetUtils.class)) { |
|
||||||
mockedNetUtils.when(NetUtils::getHost) |
|
||||||
.thenReturn(localMachine + "1"); |
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
String log = logClient.viewLog(localMachine, port, path); |
|
||||||
Assertions.assertNotNull(log); |
|
||||||
} |
|
||||||
|
|
||||||
Command command = new Command(); |
|
||||||
command.setBody(JSONUtils.toJsonString(new ViewLogResponseCommand("")).getBytes(StandardCharsets.UTF_8)); |
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
String log = logClient.viewLog(localMachine, port, path); |
|
||||||
Assertions.assertNotNull(log); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testClose() { |
|
||||||
try ( |
|
||||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
|
||||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
|
||||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
|
||||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
|
||||||
.thenReturn(remotingClient); |
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
logClient.close(); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testRollViewLog() throws Exception { |
|
||||||
try ( |
|
||||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
|
||||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
|
||||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
|
||||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
|
||||||
.thenReturn(remotingClient); |
|
||||||
Command command = new Command(); |
|
||||||
command.setBody(JSONUtils.toJsonByteArray(new RollViewLogResponseCommand("success"))); |
|
||||||
Mockito.when( |
|
||||||
remotingClient.sendSync(Mockito.any(Host.class), Mockito.any(Command.class), Mockito.anyLong())) |
|
||||||
.thenReturn(command); |
|
||||||
|
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
String msg = logClient.rollViewLog("localhost", 1234, "/tmp/log", 0, 10); |
|
||||||
Assertions.assertNotNull(msg); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testGetLogBytes() throws Exception { |
|
||||||
try ( |
|
||||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
|
||||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
|
||||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
|
||||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
|
||||||
.thenReturn(remotingClient); |
|
||||||
Command command = new Command(); |
|
||||||
command.setBody( |
|
||||||
JSONUtils.toJsonByteArray(new GetLogBytesResponseCommand("log".getBytes(StandardCharsets.UTF_8)))); |
|
||||||
Mockito.when( |
|
||||||
remotingClient.sendSync(Mockito.any(Host.class), Mockito.any(Command.class), Mockito.anyLong())) |
|
||||||
.thenReturn(command); |
|
||||||
|
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
byte[] logBytes = logClient.getLogBytes("localhost", 1234, "/tmp/log"); |
|
||||||
Assertions.assertNotNull(logBytes); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testRemoveTaskLog() { |
|
||||||
|
|
||||||
try ( |
|
||||||
MockedStatic<NettyRemotingClientFactory> mockedNettyRemotingClientFactory = |
|
||||||
Mockito.mockStatic(NettyRemotingClientFactory.class)) { |
|
||||||
NettyRemotingClient remotingClient = Mockito.mock(NettyRemotingClient.class); |
|
||||||
mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) |
|
||||||
.thenReturn(remotingClient); |
|
||||||
|
|
||||||
LogClient logClient = new LogClient(); |
|
||||||
Assertions.assertDoesNotThrow(() -> logClient.removeTaskLog(Host.of("localhost:1234"), "/log/path")); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,108 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.service.log; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.BeforeEach; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.Level; |
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent; |
|
||||||
|
|
||||||
public class TaskLogDiscriminatorTest { |
|
||||||
|
|
||||||
/** |
|
||||||
* log base |
|
||||||
*/ |
|
||||||
private String logBase = "logs"; |
|
||||||
|
|
||||||
TaskLogDiscriminator taskLogDiscriminator; |
|
||||||
|
|
||||||
@BeforeEach |
|
||||||
public void before() { |
|
||||||
taskLogDiscriminator = new TaskLogDiscriminator(); |
|
||||||
taskLogDiscriminator.setLogBase("logs"); |
|
||||||
taskLogDiscriminator.setKey("123"); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void getDiscriminatingValue() { |
|
||||||
String result = taskLogDiscriminator.getDiscriminatingValue(new LoggingEvent() { |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getThreadName() { |
|
||||||
return "taskAppId=TASK-20220105-101-1-1001"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Level getLevel() { |
|
||||||
return null; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getMessage() { |
|
||||||
return null; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Object[] getArgumentArray() { |
|
||||||
return new Object[0]; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getFormattedMessage() { |
|
||||||
return null; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getLoggerName() { |
|
||||||
return TaskConstants.TASK_LOG_LOGGER_NAME; |
|
||||||
} |
|
||||||
}); |
|
||||||
Assertions.assertEquals("20220105/101-1-1001", result); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void start() { |
|
||||||
taskLogDiscriminator.start(); |
|
||||||
Assertions.assertEquals(true, taskLogDiscriminator.isStarted()); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void getKey() { |
|
||||||
Assertions.assertEquals("123", taskLogDiscriminator.getKey()); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void setKey() { |
|
||||||
|
|
||||||
taskLogDiscriminator.setKey("123"); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void getLogBase() { |
|
||||||
Assertions.assertEquals("logs", taskLogDiscriminator.getLogBase()); |
|
||||||
} |
|
||||||
|
|
||||||
@Test |
|
||||||
public void setLogBase() { |
|
||||||
taskLogDiscriminator.setLogBase("logs"); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,71 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.service.log; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.Level; |
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent; |
|
||||||
import ch.qos.logback.core.spi.FilterReply; |
|
||||||
|
|
||||||
public class TaskLogFilterTest { |
|
||||||
|
|
||||||
@Test |
|
||||||
public void decide() { |
|
||||||
TaskLogFilter taskLogFilter = new TaskLogFilter(); |
|
||||||
|
|
||||||
FilterReply filterReply = taskLogFilter.decide(new LoggingEvent() { |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getThreadName() { |
|
||||||
return TaskConstants.TASK_APPID_LOG_FORMAT; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Level getLevel() { |
|
||||||
return Level.INFO; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getMessage() { |
|
||||||
return "raw script : echo 222"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Object[] getArgumentArray() { |
|
||||||
return new Object[0]; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getFormattedMessage() { |
|
||||||
return "raw script : echo 222"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getLoggerName() { |
|
||||||
return TaskConstants.TASK_LOG_LOGGER_NAME; |
|
||||||
} |
|
||||||
}); |
|
||||||
|
|
||||||
Assertions.assertEquals(FilterReply.ACCEPT, filterReply); |
|
||||||
|
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,73 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.utils; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.DateConstants; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator; |
|
||||||
|
|
||||||
import java.nio.file.Path; |
|
||||||
import java.nio.file.Paths; |
|
||||||
import java.util.Date; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
import org.junit.jupiter.api.extension.ExtendWith; |
|
||||||
import org.mockito.Mockito; |
|
||||||
import org.mockito.junit.jupiter.MockitoExtension; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.Logger; |
|
||||||
import ch.qos.logback.classic.sift.SiftingAppender; |
|
||||||
|
|
||||||
@ExtendWith(MockitoExtension.class) |
|
||||||
public class LogUtilsTest { |
|
||||||
|
|
||||||
@Test |
|
||||||
public void testGetTaskLogPath() { |
|
||||||
Date firstSubmitTime = new Date(); |
|
||||||
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); |
|
||||||
taskExecutionContext.setProcessInstanceId(100); |
|
||||||
taskExecutionContext.setTaskInstanceId(1000); |
|
||||||
taskExecutionContext.setProcessDefineCode(1L); |
|
||||||
taskExecutionContext.setProcessDefineVersion(1); |
|
||||||
taskExecutionContext.setFirstSubmitTime(firstSubmitTime.getTime()); |
|
||||||
|
|
||||||
Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT"); |
|
||||||
Assertions.assertNotNull(rootLogger); |
|
||||||
|
|
||||||
SiftingAppender appender = Mockito.mock(SiftingAppender.class); |
|
||||||
// it's a trick to mock logger.getAppend("TASKLOGFILE")
|
|
||||||
Mockito.when(appender.getName()).thenReturn("TASKLOGFILE"); |
|
||||||
rootLogger.addAppender(appender); |
|
||||||
|
|
||||||
Path logBase = Paths.get("path").resolve("to").resolve("test"); |
|
||||||
|
|
||||||
TaskLogDiscriminator taskLogDiscriminator = Mockito.mock(TaskLogDiscriminator.class); |
|
||||||
Mockito.when(taskLogDiscriminator.getLogBase()).thenReturn(logBase.toString()); |
|
||||||
Mockito.when(appender.getDiscriminator()).thenReturn(taskLogDiscriminator); |
|
||||||
|
|
||||||
Path logPath = Paths.get(".").toAbsolutePath().getParent() |
|
||||||
.resolve(logBase) |
|
||||||
.resolve(DateUtils.format(firstSubmitTime, DateConstants.YYYYMMDD, null)) |
|
||||||
.resolve("1_1-100-1000.log"); |
|
||||||
Assertions.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext)); |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -0,0 +1,78 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> |
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<conversionRule conversionWord="message" |
||||||
|
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||||
|
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||||
|
<filter class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogFilter"/> |
||||||
|
<Discriminator class="org.apache.dolphinscheduler.plugin.task.api.log.TaskLogDiscriminator"> |
||||||
|
<key>taskInstanceLogFullPath</key> |
||||||
|
<logBase>${log.base}</logBase> |
||||||
|
</Discriminator> |
||||||
|
<sift> |
||||||
|
<appender name="FILE-${taskInstanceLogFullPath}" class="ch.qos.logback.core.FileAppender"> |
||||||
|
<file>${taskInstanceLogFullPath}</file> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %message%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
<append>true</append> |
||||||
|
</appender> |
||||||
|
</sift> |
||||||
|
</appender> |
||||||
|
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-worker.log</file> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>200MB</maxFileSize> |
||||||
|
<totalSizeCap>50GB</totalSizeCap> |
||||||
|
<cleanHistoryOnStart>true</cleanHistoryOnStart> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<if condition="${DOCKER:-false}"> |
||||||
|
<then> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
<appender-ref ref="TASKLOGFILE"/> |
||||||
|
<appender-ref ref="WORKERLOGFILE"/> |
||||||
|
</root> |
||||||
|
</configuration> |
||||||
Loading…
Reference in new issue