Browse Source
* combine logback config of master/worker/alert/api server to one logback.xmlpull/2/head
dailidong
5 years ago
committed by
GitHub
31 changed files with 955 additions and 370 deletions
@ -1,49 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
<property name="log.base" value="logs" /> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-alert.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>20</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="ALERTLOGFILE"/> |
||||
</root> |
||||
</configuration> |
||||
@ -1,60 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> |
||||
<logger name="org.apache.zookeeper" level="WARN"/> |
||||
<logger name="org.apache.hbase" level="WARN"/> |
||||
<logger name="org.apache.hadoop" level="WARN"/> |
||||
|
||||
<property name="log.base" value="logs" /> |
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<appender name="APISERVERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<!-- Log level filter --> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<file>${log.base}/dolphinscheduler-api-server.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
|
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
|
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="APISERVERLOGFILE" /> |
||||
</root> |
||||
</configuration> |
||||
@ -0,0 +1,169 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
|
||||
<property name="log.base" value="logs"/> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
|
||||
<!-- master server logback config start --> |
||||
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-master.log</file> |
||||
<!--<filter class="org.apache.dolphinscheduler.common.log.MasterLogFilter"> |
||||
<level>INFO</level> |
||||
</filter>--> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- master server logback config end --> |
||||
|
||||
|
||||
<!-- worker server logback config start --> |
||||
<conversionRule conversionWord="messsage" |
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<filter class="org.apache.dolphinscheduler.common.log.TaskLogFilter"/> |
||||
<Discriminator class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator"> |
||||
<key>taskAppId</key> |
||||
<logBase>${log.base}</logBase> |
||||
</Discriminator> |
||||
<sift> |
||||
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender"> |
||||
<file>${log.base}/${taskAppId}.log</file> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
<append>true</append> |
||||
</appender> |
||||
</sift> |
||||
</appender> |
||||
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-worker.log</file> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter"/> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- worker server logback config end --> |
||||
|
||||
|
||||
<!-- alert server logback config start --> |
||||
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-alert.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>20</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- alert server logback config end --> |
||||
|
||||
|
||||
<!-- api server logback config start --> |
||||
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-api-server.log</file> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- api server logback config end --> |
||||
|
||||
<logger name="org.apache.zookeeper" level="WARN"/> |
||||
<logger name="org.apache.hbase" level="WARN"/> |
||||
<logger name="org.apache.hadoop" level="WARN"/> |
||||
|
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="STDOUT"/> |
||||
|
||||
<if condition='p("server").contains("master-server")'> |
||||
<then> |
||||
<appender-ref ref="MASTERLOGFILE"/> |
||||
</then> |
||||
</if> |
||||
<if condition='p("server").contains("worker-server")'> |
||||
<then> |
||||
<appender-ref ref="TASKLOGFILE"/> |
||||
<appender-ref ref="WORKERLOGFILE"/> |
||||
</then> |
||||
</if> |
||||
<if condition='p("server").contains("alert-server")'> |
||||
<then> |
||||
<appender-ref ref="ALERTLOGFILE"/> |
||||
</then> |
||||
</if> |
||||
<if condition='p("server").contains("api-server")'> |
||||
<then> |
||||
<appender-ref ref="APILOGFILE"/> |
||||
</then> |
||||
</if> |
||||
</root> |
||||
|
||||
</configuration> |
||||
@ -0,0 +1,118 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.log; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||
import ch.qos.logback.core.spi.FilterReply; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Marker; |
||||
import java.util.Map; |
||||
|
||||
public class MasterLogFilterTest { |
||||
|
||||
@Test |
||||
public void decide() { |
||||
MasterLogFilter masterLogFilter = new MasterLogFilter(); |
||||
|
||||
|
||||
FilterReply filterReply = masterLogFilter.decide(new ILoggingEvent() { |
||||
@Override |
||||
public String getThreadName() { |
||||
return Constants.THREAD_NAME_MASTER_SERVER; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return Level.INFO; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return "master insert into queue success, task : shell2"; |
||||
// return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed";
|
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return "master insert into queue success, task : shell2"; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public LoggerContextVO getLoggerContextVO() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public IThrowableProxy getThrowableProxy() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public StackTraceElement[] getCallerData() { |
||||
return new StackTraceElement[0]; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasCallerData() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Marker getMarker() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMDCPropertyMap() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMdc() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public long getTimeStamp() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void prepareForDeferredProcessing() { |
||||
|
||||
} |
||||
}); |
||||
|
||||
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,179 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.log; |
||||
|
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.slf4j.Marker; |
||||
|
||||
import java.util.Map; |
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
|
||||
public class SensitiveDataConverterTest { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); |
||||
|
||||
/** |
||||
* password pattern |
||||
*/ |
||||
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); |
||||
|
||||
private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||
"\"database\":\"carbond\"," + |
||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||
"\"user\":\"view\"," + |
||||
"\"password\":\"view1\"}"; |
||||
|
||||
private final String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||
"\"database\":\"carbond\"," + |
||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||
"\"user\":\"view\"," + |
||||
"\"password\":\"******\"}"; |
||||
@Test |
||||
public void convert() { |
||||
SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter(); |
||||
String result = sensitiveDataConverter.convert(new ILoggingEvent() { |
||||
@Override |
||||
public String getThreadName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return Level.INFO; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return logMsg; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public LoggerContextVO getLoggerContextVO() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public IThrowableProxy getThrowableProxy() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public StackTraceElement[] getCallerData() { |
||||
return new StackTraceElement[0]; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasCallerData() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Marker getMarker() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMDCPropertyMap() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMdc() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public long getTimeStamp() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void prepareForDeferredProcessing() { |
||||
|
||||
} |
||||
}); |
||||
|
||||
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* mask sensitive logMsg - sql task datasource password |
||||
*/ |
||||
@Test |
||||
public void testPwdLogMsgConverter() { |
||||
logger.info("parameter : {}", logMsg); |
||||
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); |
||||
|
||||
Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); |
||||
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* password regex test |
||||
* |
||||
* @param logMsg original log |
||||
*/ |
||||
private static String passwordHandler(Pattern pattern, String logMsg) { |
||||
|
||||
Matcher matcher = pattern.matcher(logMsg); |
||||
|
||||
StringBuffer sb = new StringBuffer(logMsg.length()); |
||||
|
||||
while (matcher.find()) { |
||||
|
||||
String password = matcher.group(); |
||||
|
||||
String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password); |
||||
|
||||
matcher.appendReplacement(sb, maskPassword); |
||||
} |
||||
matcher.appendTail(sb); |
||||
|
||||
return sb.toString(); |
||||
} |
||||
|
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,155 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.log; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.slf4j.Marker; |
||||
|
||||
import java.util.Map; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class TaskLogDiscriminatorTest { |
||||
|
||||
/** |
||||
* log base |
||||
*/ |
||||
private String logBase = "logs"; |
||||
|
||||
TaskLogDiscriminator taskLogDiscriminator; |
||||
|
||||
@Before |
||||
public void before(){ |
||||
taskLogDiscriminator = new TaskLogDiscriminator(); |
||||
taskLogDiscriminator.setLogBase("logs"); |
||||
taskLogDiscriminator.setKey("123"); |
||||
} |
||||
|
||||
@Test |
||||
public void getDiscriminatingValue() { |
||||
String result = taskLogDiscriminator.getDiscriminatingValue(new ILoggingEvent() { |
||||
@Override |
||||
public String getThreadName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return "[taskAppId=TASK-1-1-1"; |
||||
} |
||||
|
||||
@Override |
||||
public LoggerContextVO getLoggerContextVO() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public IThrowableProxy getThrowableProxy() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public StackTraceElement[] getCallerData() { |
||||
return new StackTraceElement[0]; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasCallerData() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Marker getMarker() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMDCPropertyMap() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMdc() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public long getTimeStamp() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void prepareForDeferredProcessing() { |
||||
|
||||
} |
||||
}); |
||||
Assert.assertEquals("1/1/", result); |
||||
} |
||||
|
||||
@Test |
||||
public void start() { |
||||
taskLogDiscriminator.start(); |
||||
Assert.assertEquals(true, taskLogDiscriminator.isStarted()); |
||||
} |
||||
|
||||
@Test |
||||
public void getKey() { |
||||
Assert.assertEquals("123", taskLogDiscriminator.getKey()); |
||||
} |
||||
|
||||
@Test |
||||
public void setKey() { |
||||
|
||||
taskLogDiscriminator.setKey("123"); |
||||
} |
||||
|
||||
@Test |
||||
public void getLogBase() { |
||||
Assert.assertEquals("logs", taskLogDiscriminator.getLogBase()); |
||||
} |
||||
|
||||
@Test |
||||
public void setLogBase() { |
||||
taskLogDiscriminator.setLogBase("logs"); |
||||
} |
||||
} |
||||
@ -0,0 +1,120 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.log; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||
import ch.qos.logback.core.spi.FilterReply; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Marker; |
||||
|
||||
import java.util.Map; |
||||
|
||||
|
||||
public class TaskLogFilterTest { |
||||
|
||||
@Test |
||||
public void decide() { |
||||
TaskLogFilter taskLogFilter = new TaskLogFilter(); |
||||
|
||||
|
||||
FilterReply filterReply = taskLogFilter.decide(new ILoggingEvent() { |
||||
@Override |
||||
public String getThreadName() { |
||||
return LoggerUtils.TASK_LOGGER_THREAD_NAME; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return Level.INFO; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return "raw script : echo 222"; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return "raw script : echo 222"; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public LoggerContextVO getLoggerContextVO() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public IThrowableProxy getThrowableProxy() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public StackTraceElement[] getCallerData() { |
||||
return new StackTraceElement[0]; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasCallerData() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Marker getMarker() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMDCPropertyMap() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMdc() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public long getTimeStamp() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void prepareForDeferredProcessing() { |
||||
|
||||
} |
||||
}); |
||||
|
||||
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,119 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.log; |
||||
|
||||
import ch.qos.logback.classic.Level; |
||||
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||
import ch.qos.logback.core.spi.FilterReply; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Marker; |
||||
|
||||
import java.util.Map; |
||||
|
||||
|
||||
public class WorkerLogFilterTest { |
||||
|
||||
@Test |
||||
public void decide() { |
||||
WorkerLogFilter workerLogFilter = new WorkerLogFilter(); |
||||
|
||||
|
||||
FilterReply filterReply = workerLogFilter.decide(new ILoggingEvent() { |
||||
@Override |
||||
public String getThreadName() { |
||||
return Constants.THREAD_NAME_WORKER_SERVER; |
||||
} |
||||
|
||||
@Override |
||||
public Level getLevel() { |
||||
return Level.INFO; |
||||
} |
||||
|
||||
@Override |
||||
public String getMessage() { |
||||
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
||||
} |
||||
|
||||
@Override |
||||
public Object[] getArgumentArray() { |
||||
return new Object[0]; |
||||
} |
||||
|
||||
@Override |
||||
public String getFormattedMessage() { |
||||
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
||||
} |
||||
|
||||
@Override |
||||
public String getLoggerName() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public LoggerContextVO getLoggerContextVO() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public IThrowableProxy getThrowableProxy() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public StackTraceElement[] getCallerData() { |
||||
return new StackTraceElement[0]; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasCallerData() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Marker getMarker() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMDCPropertyMap() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, String> getMdc() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
public long getTimeStamp() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void prepareForDeferredProcessing() { |
||||
|
||||
} |
||||
}); |
||||
|
||||
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.utils; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.List; |
||||
|
||||
public class LoggerUtilsTest { |
||||
private Logger logger = LoggerFactory.getLogger(LoggerUtilsTest.class); |
||||
|
||||
@Test |
||||
public void buildTaskId() { |
||||
|
||||
String taskId = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,79,4084,15210); |
||||
|
||||
Assert.assertEquals(" - [taskAppId=TASK-79-4084-15210]", taskId); |
||||
} |
||||
|
||||
@Test |
||||
public void getAppIds() { |
||||
List<String> appIdList = LoggerUtils.getAppIds("Running job: application_1_1",logger); |
||||
Assert.assertEquals("application_1_1", appIdList.get(0)); |
||||
|
||||
} |
||||
} |
||||
@ -1,52 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
<property name="log.base" value="logs" /> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-master.log</file> |
||||
<filter class="org.apache.dolphinscheduler.server.master.log.MasterLogFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="MASTERLOGFILE"/> |
||||
</root> |
||||
</configuration> |
||||
@ -1,92 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.worker.log; |
||||
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
|
||||
public class SensitiveDataConverterTest { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); |
||||
|
||||
/** |
||||
* password pattern |
||||
*/ |
||||
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); |
||||
|
||||
|
||||
/** |
||||
* mask sensitive logMsg - sql task datasource password |
||||
*/ |
||||
@Test |
||||
public void testPwdLogMsgConverter() { |
||||
|
||||
String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||
"\"database\":\"carbond\"," + |
||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||
"\"user\":\"view\"," + |
||||
"\"password\":\"view1\"}"; |
||||
|
||||
String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||
"\"database\":\"carbond\"," + |
||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||
"\"user\":\"view\"," + |
||||
"\"password\":\"******\"}"; |
||||
|
||||
|
||||
logger.info("parameter : {}", logMsg); |
||||
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); |
||||
|
||||
Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); |
||||
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* password regex test |
||||
* |
||||
* @param logMsg original log |
||||
*/ |
||||
private static String passwordHandler(Pattern pattern, String logMsg) { |
||||
|
||||
Matcher matcher = pattern.matcher(logMsg); |
||||
|
||||
StringBuffer sb = new StringBuffer(logMsg.length()); |
||||
|
||||
while (matcher.find()) { |
||||
|
||||
String password = matcher.group(); |
||||
|
||||
String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); |
||||
|
||||
matcher.appendReplacement(sb, maskPassword); |
||||
} |
||||
matcher.appendTail(sb); |
||||
|
||||
return sb.toString(); |
||||
} |
||||
|
||||
|
||||
} |
||||
Loading…
Reference in new issue