Browse Source
* combine logback config of master/worker/alert/api server to one logback.xmlpull/2/head
dailidong
5 years ago
committed by
GitHub
31 changed files with 955 additions and 370 deletions
@ -1,49 +0,0 @@ |
|||||||
<?xml version="1.0" encoding="UTF-8"?> |
|
||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
|
|
||||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
|
||||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
|
||||||
<property name="log.base" value="logs" /> |
|
||||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
</appender> |
|
||||||
|
|
||||||
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
|
||||||
<file>${log.base}/dolphinscheduler-alert.log</file> |
|
||||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
|
||||||
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
|
||||||
<maxHistory>20</maxHistory> |
|
||||||
<maxFileSize>64MB</maxFileSize> |
|
||||||
</rollingPolicy> |
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
</appender> |
|
||||||
|
|
||||||
<root level="INFO"> |
|
||||||
<appender-ref ref="ALERTLOGFILE"/> |
|
||||||
</root> |
|
||||||
</configuration> |
|
||||||
@ -1,60 +0,0 @@ |
|||||||
<?xml version="1.0" encoding="UTF-8"?> |
|
||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
|
|
||||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
|
||||||
<configuration scan="true" scanPeriod="120 seconds"> |
|
||||||
<logger name="org.apache.zookeeper" level="WARN"/> |
|
||||||
<logger name="org.apache.hbase" level="WARN"/> |
|
||||||
<logger name="org.apache.hadoop" level="WARN"/> |
|
||||||
|
|
||||||
<property name="log.base" value="logs" /> |
|
||||||
|
|
||||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
</appender> |
|
||||||
|
|
||||||
<appender name="APISERVERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
|
||||||
<!-- Log level filter --> |
|
||||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
|
||||||
<level>INFO</level> |
|
||||||
</filter> |
|
||||||
<file>${log.base}/dolphinscheduler-api-server.log</file> |
|
||||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
|
||||||
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
|
||||||
<maxHistory>168</maxHistory> |
|
||||||
<maxFileSize>64MB</maxFileSize> |
|
||||||
</rollingPolicy> |
|
||||||
|
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
|
|
||||||
</appender> |
|
||||||
|
|
||||||
<root level="INFO"> |
|
||||||
<appender-ref ref="APISERVERLOGFILE" /> |
|
||||||
</root> |
|
||||||
</configuration> |
|
||||||
@ -0,0 +1,169 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||||
|
|
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
|
||||||
|
<!-- master server logback config start --> |
||||||
|
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-master.log</file> |
||||||
|
<!--<filter class="org.apache.dolphinscheduler.common.log.MasterLogFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter>--> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>200MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- master server logback config end --> |
||||||
|
|
||||||
|
|
||||||
|
<!-- worker server logback config start --> |
||||||
|
<conversionRule conversionWord="messsage" |
||||||
|
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> |
||||||
|
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter> |
||||||
|
<filter class="org.apache.dolphinscheduler.common.log.TaskLogFilter"/> |
||||||
|
<Discriminator class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator"> |
||||||
|
<key>taskAppId</key> |
||||||
|
<logBase>${log.base}</logBase> |
||||||
|
</Discriminator> |
||||||
|
<sift> |
||||||
|
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender"> |
||||||
|
<file>${log.base}/${taskAppId}.log</file> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
<append>true</append> |
||||||
|
</appender> |
||||||
|
</sift> |
||||||
|
</appender> |
||||||
|
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-worker.log</file> |
||||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter> |
||||||
|
<filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter"/> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>200MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- worker server logback config end --> |
||||||
|
|
||||||
|
|
||||||
|
<!-- alert server logback config start --> |
||||||
|
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-alert.log</file> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>20</maxHistory> |
||||||
|
<maxFileSize>64MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- alert server logback config end --> |
||||||
|
|
||||||
|
|
||||||
|
<!-- api server logback config start --> |
||||||
|
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-api-server.log</file> |
||||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>64MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- api server logback config end --> |
||||||
|
|
||||||
|
<logger name="org.apache.zookeeper" level="WARN"/> |
||||||
|
<logger name="org.apache.hbase" level="WARN"/> |
||||||
|
<logger name="org.apache.hadoop" level="WARN"/> |
||||||
|
|
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
|
||||||
|
<if condition='p("server").contains("master-server")'> |
||||||
|
<then> |
||||||
|
<appender-ref ref="MASTERLOGFILE"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
<if condition='p("server").contains("worker-server")'> |
||||||
|
<then> |
||||||
|
<appender-ref ref="TASKLOGFILE"/> |
||||||
|
<appender-ref ref="WORKERLOGFILE"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
<if condition='p("server").contains("alert-server")'> |
||||||
|
<then> |
||||||
|
<appender-ref ref="ALERTLOGFILE"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
<if condition='p("server").contains("api-server")'> |
||||||
|
<then> |
||||||
|
<appender-ref ref="APILOGFILE"/> |
||||||
|
</then> |
||||||
|
</if> |
||||||
|
</root> |
||||||
|
|
||||||
|
</configuration> |
||||||
@ -0,0 +1,118 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.log; |
||||||
|
|
||||||
|
import ch.qos.logback.classic.Level; |
||||||
|
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||||
|
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||||
|
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||||
|
import ch.qos.logback.core.spi.FilterReply; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Marker; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
public class MasterLogFilterTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void decide() { |
||||||
|
MasterLogFilter masterLogFilter = new MasterLogFilter(); |
||||||
|
|
||||||
|
|
||||||
|
FilterReply filterReply = masterLogFilter.decide(new ILoggingEvent() { |
||||||
|
@Override |
||||||
|
public String getThreadName() { |
||||||
|
return Constants.THREAD_NAME_MASTER_SERVER; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Level getLevel() { |
||||||
|
return Level.INFO; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getMessage() { |
||||||
|
return "master insert into queue success, task : shell2"; |
||||||
|
// return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed";
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object[] getArgumentArray() { |
||||||
|
return new Object[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getFormattedMessage() { |
||||||
|
return "master insert into queue success, task : shell2"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getLoggerName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public LoggerContextVO getLoggerContextVO() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public IThrowableProxy getThrowableProxy() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StackTraceElement[] getCallerData() { |
||||||
|
return new StackTraceElement[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasCallerData() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Marker getMarker() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMDCPropertyMap() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMdc() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getTimeStamp() { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void prepareForDeferredProcessing() { |
||||||
|
|
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,179 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.log; |
||||||
|
|
||||||
|
|
||||||
|
import ch.qos.logback.classic.Level; |
||||||
|
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||||
|
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||||
|
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.slf4j.Marker; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
import java.util.regex.Matcher; |
||||||
|
import java.util.regex.Pattern; |
||||||
|
|
||||||
|
public class SensitiveDataConverterTest { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* password pattern |
||||||
|
*/ |
||||||
|
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); |
||||||
|
|
||||||
|
private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||||
|
"\"database\":\"carbond\"," + |
||||||
|
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||||
|
"\"user\":\"view\"," + |
||||||
|
"\"password\":\"view1\"}"; |
||||||
|
|
||||||
|
private final String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
||||||
|
"\"database\":\"carbond\"," + |
||||||
|
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
||||||
|
"\"user\":\"view\"," + |
||||||
|
"\"password\":\"******\"}"; |
||||||
|
@Test |
||||||
|
public void convert() { |
||||||
|
SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter(); |
||||||
|
String result = sensitiveDataConverter.convert(new ILoggingEvent() { |
||||||
|
@Override |
||||||
|
public String getThreadName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Level getLevel() { |
||||||
|
return Level.INFO; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getMessage() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object[] getArgumentArray() { |
||||||
|
return new Object[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getFormattedMessage() { |
||||||
|
return logMsg; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getLoggerName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public LoggerContextVO getLoggerContextVO() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public IThrowableProxy getThrowableProxy() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StackTraceElement[] getCallerData() { |
||||||
|
return new StackTraceElement[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasCallerData() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Marker getMarker() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMDCPropertyMap() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMdc() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getTimeStamp() { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void prepareForDeferredProcessing() { |
||||||
|
|
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* mask sensitive logMsg - sql task datasource password |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testPwdLogMsgConverter() { |
||||||
|
logger.info("parameter : {}", logMsg); |
||||||
|
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); |
||||||
|
|
||||||
|
Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); |
||||||
|
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* password regex test |
||||||
|
* |
||||||
|
* @param logMsg original log |
||||||
|
*/ |
||||||
|
private static String passwordHandler(Pattern pattern, String logMsg) { |
||||||
|
|
||||||
|
Matcher matcher = pattern.matcher(logMsg); |
||||||
|
|
||||||
|
StringBuffer sb = new StringBuffer(logMsg.length()); |
||||||
|
|
||||||
|
while (matcher.find()) { |
||||||
|
|
||||||
|
String password = matcher.group(); |
||||||
|
|
||||||
|
String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password); |
||||||
|
|
||||||
|
matcher.appendReplacement(sb, maskPassword); |
||||||
|
} |
||||||
|
matcher.appendTail(sb); |
||||||
|
|
||||||
|
return sb.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,155 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.log; |
||||||
|
|
||||||
|
import ch.qos.logback.classic.Level; |
||||||
|
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||||
|
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||||
|
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Marker; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class TaskLogDiscriminatorTest { |
||||||
|
|
||||||
|
/** |
||||||
|
* log base |
||||||
|
*/ |
||||||
|
private String logBase = "logs"; |
||||||
|
|
||||||
|
TaskLogDiscriminator taskLogDiscriminator; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before(){ |
||||||
|
taskLogDiscriminator = new TaskLogDiscriminator(); |
||||||
|
taskLogDiscriminator.setLogBase("logs"); |
||||||
|
taskLogDiscriminator.setKey("123"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getDiscriminatingValue() { |
||||||
|
String result = taskLogDiscriminator.getDiscriminatingValue(new ILoggingEvent() { |
||||||
|
@Override |
||||||
|
public String getThreadName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Level getLevel() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getMessage() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object[] getArgumentArray() { |
||||||
|
return new Object[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getFormattedMessage() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getLoggerName() { |
||||||
|
return "[taskAppId=TASK-1-1-1"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public LoggerContextVO getLoggerContextVO() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public IThrowableProxy getThrowableProxy() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StackTraceElement[] getCallerData() { |
||||||
|
return new StackTraceElement[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasCallerData() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Marker getMarker() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMDCPropertyMap() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMdc() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getTimeStamp() { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void prepareForDeferredProcessing() { |
||||||
|
|
||||||
|
} |
||||||
|
}); |
||||||
|
Assert.assertEquals("1/1/", result); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void start() { |
||||||
|
taskLogDiscriminator.start(); |
||||||
|
Assert.assertEquals(true, taskLogDiscriminator.isStarted()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getKey() { |
||||||
|
Assert.assertEquals("123", taskLogDiscriminator.getKey()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void setKey() { |
||||||
|
|
||||||
|
taskLogDiscriminator.setKey("123"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getLogBase() { |
||||||
|
Assert.assertEquals("logs", taskLogDiscriminator.getLogBase()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void setLogBase() { |
||||||
|
taskLogDiscriminator.setLogBase("logs"); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,120 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.log; |
||||||
|
|
||||||
|
import ch.qos.logback.classic.Level; |
||||||
|
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||||
|
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||||
|
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||||
|
import ch.qos.logback.core.spi.FilterReply; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Marker; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
|
||||||
|
public class TaskLogFilterTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void decide() { |
||||||
|
TaskLogFilter taskLogFilter = new TaskLogFilter(); |
||||||
|
|
||||||
|
|
||||||
|
FilterReply filterReply = taskLogFilter.decide(new ILoggingEvent() { |
||||||
|
@Override |
||||||
|
public String getThreadName() { |
||||||
|
return LoggerUtils.TASK_LOGGER_THREAD_NAME; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Level getLevel() { |
||||||
|
return Level.INFO; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getMessage() { |
||||||
|
return "raw script : echo 222"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object[] getArgumentArray() { |
||||||
|
return new Object[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getFormattedMessage() { |
||||||
|
return "raw script : echo 222"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getLoggerName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public LoggerContextVO getLoggerContextVO() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public IThrowableProxy getThrowableProxy() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StackTraceElement[] getCallerData() { |
||||||
|
return new StackTraceElement[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasCallerData() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Marker getMarker() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMDCPropertyMap() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMdc() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getTimeStamp() { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void prepareForDeferredProcessing() { |
||||||
|
|
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,119 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.log; |
||||||
|
|
||||||
|
import ch.qos.logback.classic.Level; |
||||||
|
import ch.qos.logback.classic.spi.ILoggingEvent; |
||||||
|
import ch.qos.logback.classic.spi.IThrowableProxy; |
||||||
|
import ch.qos.logback.classic.spi.LoggerContextVO; |
||||||
|
import ch.qos.logback.core.spi.FilterReply; |
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Marker; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
|
||||||
|
public class WorkerLogFilterTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void decide() { |
||||||
|
WorkerLogFilter workerLogFilter = new WorkerLogFilter(); |
||||||
|
|
||||||
|
|
||||||
|
FilterReply filterReply = workerLogFilter.decide(new ILoggingEvent() { |
||||||
|
@Override |
||||||
|
public String getThreadName() { |
||||||
|
return Constants.THREAD_NAME_WORKER_SERVER; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Level getLevel() { |
||||||
|
return Level.INFO; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getMessage() { |
||||||
|
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Object[] getArgumentArray() { |
||||||
|
return new Object[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getFormattedMessage() { |
||||||
|
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getLoggerName() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public LoggerContextVO getLoggerContextVO() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public IThrowableProxy getThrowableProxy() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StackTraceElement[] getCallerData() { |
||||||
|
return new StackTraceElement[0]; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasCallerData() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Marker getMarker() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMDCPropertyMap() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, String> getMdc() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getTimeStamp() { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void prepareForDeferredProcessing() { |
||||||
|
|
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
Assert.assertEquals(FilterReply.ACCEPT, filterReply); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,43 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.utils; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class LoggerUtilsTest { |
||||||
|
private Logger logger = LoggerFactory.getLogger(LoggerUtilsTest.class); |
||||||
|
|
||||||
|
@Test |
||||||
|
public void buildTaskId() { |
||||||
|
|
||||||
|
String taskId = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,79,4084,15210); |
||||||
|
|
||||||
|
Assert.assertEquals(" - [taskAppId=TASK-79-4084-15210]", taskId); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getAppIds() { |
||||||
|
List<String> appIdList = LoggerUtils.getAppIds("Running job: application_1_1",logger); |
||||||
|
Assert.assertEquals("application_1_1", appIdList.get(0)); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
@ -1,52 +0,0 @@ |
|||||||
<?xml version="1.0" encoding="UTF-8"?> |
|
||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
|
|
||||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
|
||||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
|
||||||
<property name="log.base" value="logs" /> |
|
||||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
</appender> |
|
||||||
|
|
||||||
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
|
||||||
<file>${log.base}/dolphinscheduler-master.log</file> |
|
||||||
<filter class="org.apache.dolphinscheduler.server.master.log.MasterLogFilter"> |
|
||||||
<level>INFO</level> |
|
||||||
</filter> |
|
||||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
|
||||||
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
|
||||||
<maxHistory>168</maxHistory> |
|
||||||
<maxFileSize>200MB</maxFileSize> |
|
||||||
</rollingPolicy> |
|
||||||
<encoder> |
|
||||||
<pattern> |
|
||||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
|
||||||
</pattern> |
|
||||||
<charset>UTF-8</charset> |
|
||||||
</encoder> |
|
||||||
</appender> |
|
||||||
|
|
||||||
<root level="INFO"> |
|
||||||
<appender-ref ref="MASTERLOGFILE"/> |
|
||||||
</root> |
|
||||||
</configuration> |
|
||||||
@ -1,92 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.server.worker.log; |
|
||||||
|
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; |
|
||||||
import org.junit.Assert; |
|
||||||
import org.junit.Test; |
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
import java.util.regex.Matcher; |
|
||||||
import java.util.regex.Pattern; |
|
||||||
|
|
||||||
public class SensitiveDataConverterTest { |
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); |
|
||||||
|
|
||||||
/** |
|
||||||
* password pattern |
|
||||||
*/ |
|
||||||
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); |
|
||||||
|
|
||||||
|
|
||||||
/** |
|
||||||
* mask sensitive logMsg - sql task datasource password |
|
||||||
*/ |
|
||||||
@Test |
|
||||||
public void testPwdLogMsgConverter() { |
|
||||||
|
|
||||||
String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
|
||||||
"\"database\":\"carbond\"," + |
|
||||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
|
||||||
"\"user\":\"view\"," + |
|
||||||
"\"password\":\"view1\"}"; |
|
||||||
|
|
||||||
String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + |
|
||||||
"\"database\":\"carbond\"," + |
|
||||||
"\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + |
|
||||||
"\"user\":\"view\"," + |
|
||||||
"\"password\":\"******\"}"; |
|
||||||
|
|
||||||
|
|
||||||
logger.info("parameter : {}", logMsg); |
|
||||||
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); |
|
||||||
|
|
||||||
Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); |
|
||||||
Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); |
|
||||||
|
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* password regex test |
|
||||||
* |
|
||||||
* @param logMsg original log |
|
||||||
*/ |
|
||||||
private static String passwordHandler(Pattern pattern, String logMsg) { |
|
||||||
|
|
||||||
Matcher matcher = pattern.matcher(logMsg); |
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer(logMsg.length()); |
|
||||||
|
|
||||||
while (matcher.find()) { |
|
||||||
|
|
||||||
String password = matcher.group(); |
|
||||||
|
|
||||||
String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); |
|
||||||
|
|
||||||
matcher.appendReplacement(sb, maskPassword); |
|
||||||
} |
|
||||||
matcher.appendTail(sb); |
|
||||||
|
|
||||||
return sb.toString(); |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
} |
|
||||||
Loading…
Reference in new issue