Browse Source

[Feature-10498] Mask the password in the log of sqoop task (#11589)

3.2.0-release
rickchengx 2 years ago committed by GitHub
parent
commit
38b876733c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      docs/docs/en/architecture/design.md
  2. 2
      docs/docs/zh/architecture/design.md
  3. 47
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java
  4. 27
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
  5. 2
      dolphinscheduler-master/src/main/resources/logback-spring.xml
  6. 2
      dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
  7. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
  8. 3
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java
  9. 38
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java
  10. 2
      dolphinscheduler-worker/src/main/resources/logback-spring.xml

2
docs/docs/en/architecture/design.md

@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc
- For details, please refer to the logback configuration of Master and Worker, as shown in the following example:
```xml
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

2
docs/docs/zh/architecture/design.md

@ -195,7 +195,7 @@
- 详情可参考Master和Worker的logback配置,如下示例:
```xml
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

47
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java

@ -15,11 +15,15 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.log;
package org.apache.dolphinscheduler.common.log;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -33,10 +37,9 @@ import com.google.common.base.Strings;
*/
public class SensitiveDataConverter extends MessageConverter {
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
private static Pattern multilinePattern;
private static HashSet<String> maskPatterns =
new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX));
@Override
public String convert(ILoggingEvent event) {
@ -45,41 +48,25 @@ public class SensitiveDataConverter extends MessageConverter {
String requestLogMsg = event.getFormattedMessage();
// desensitization log
return convertMsg(requestLogMsg);
return maskSensitiveData(requestLogMsg);
}
/**
* deal with sensitive log
*
* @param oriLogMsg original log
*/
private String convertMsg(final String oriLogMsg) {
String tempLogMsg = oriLogMsg;
if (!Strings.isNullOrEmpty(tempLogMsg)) {
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
}
return tempLogMsg;
public static void addMaskPattern(String maskPattern) {
maskPatterns.add(maskPattern);
}
/**
* password regex
*
* @param logMsg original log
*/
static String passwordHandler(Pattern pwdPattern, String logMsg) {
Matcher matcher = pwdPattern.matcher(logMsg);
public static String maskSensitiveData(final String logMsg) {
if (StringUtils.isEmpty(logMsg)) {
return logMsg;
}
multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE);
StringBuffer sb = new StringBuffer(logMsg.length());
Matcher matcher = multilinePattern.matcher(logMsg);
while (matcher.find()) {
String password = matcher.group();
String maskPassword = Strings.repeat(Constants.STAR, password.length());
matcher.appendReplacement(sb, maskPassword);
}
matcher.appendTail(sb);

27
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java → dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java

@ -15,13 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.log;
import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import java.util.regex.Pattern;
package org.apache.dolphinscheduler.common.log;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -32,11 +26,6 @@ public class SensitiveDataConverterTest {
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\","
+ "\"database\":\"carbond\","
+ "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\","
@ -49,21 +38,17 @@ public class SensitiveDataConverterTest {
+ "\"user\":\"view\","
+ "\"password\":\"*****\"}";
@Test
public void convert() {
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
}
/**
* mask sensitive logMsg - sql task datasource password
*/
@Test
public void testPwdLogMsgConverter() {
logger.info("parameter : {}", logMsg);
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
final String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg);
logger.info("original parameter : {}", logMsg);
logger.info("masked parameter : {}", maskedLog);
Assertions.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg));
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
Assertions.assertEquals(maskLogMsg, maskedLog);
}

2
dolphinscheduler-master/src/main/resources/logback-spring.xml

@ -28,7 +28,7 @@
</appender>
<conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

2
dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml

@ -48,7 +48,7 @@
<logger name="org.apache.hadoop" level="WARN"/>
<conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

1
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java

@ -72,4 +72,5 @@ public final class SqoopConstants {
public static final String UPDATE_KEY = "--update-key";
public static final String UPDATE_MODE = "--update-mode";
public static final String SQOOP_PASSWORD_REGEX = "(?<=(--password \")).+?(?=\")";
}

3
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.sqoop;
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@ -67,6 +68,8 @@ public class SqoopTask extends AbstractYarnTask {
sqoopTaskExecutionContext =
sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
}
@Override

38
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.sqoop;
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class SqoopTaskTest {
@Test
public void testSqoopPasswordMask() {
final String originalScript =
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile";
final String maskScript =
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile";
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript));
}
}

2
dolphinscheduler-worker/src/main/resources/logback-spring.xml

@ -29,7 +29,7 @@
</appender>
<conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

Loading…
Cancel
Save