Browse Source

[Feature-10498] Mask the password in the log of sqoop task (#11589)

3.2.0-release
rickchengx 2 years ago committed by GitHub
parent
commit
38b876733c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      docs/docs/en/architecture/design.md
  2. 2
      docs/docs/zh/architecture/design.md
  3. 47
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java
  4. 27
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
  5. 2
      dolphinscheduler-master/src/main/resources/logback-spring.xml
  6. 2
      dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
  7. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
  8. 3
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java
  9. 38
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java
  10. 2
      dolphinscheduler-worker/src/main/resources/logback-spring.xml

2
docs/docs/en/architecture/design.md

@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc
- For details, please refer to the logback configuration of Master and Worker, as shown in the following example: - For details, please refer to the logback configuration of Master and Worker, as shown in the following example:
```xml ```xml
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/> <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/> <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator"> <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

2
docs/docs/zh/architecture/design.md

@ -195,7 +195,7 @@
- 详情可参考Master和Worker的logback配置,如下示例: - 详情可参考Master和Worker的logback配置,如下示例:
```xml ```xml
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/> <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/> <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator"> <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

47
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java

@ -15,11 +15,15 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.log; package org.apache.dolphinscheduler.common.log;
import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -33,10 +37,9 @@ import com.google.common.base.Strings;
*/ */
public class SensitiveDataConverter extends MessageConverter { public class SensitiveDataConverter extends MessageConverter {
/** private static Pattern multilinePattern;
* password pattern private static HashSet<String> maskPatterns =
*/ new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX));
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
@Override @Override
public String convert(ILoggingEvent event) { public String convert(ILoggingEvent event) {
@ -45,41 +48,25 @@ public class SensitiveDataConverter extends MessageConverter {
String requestLogMsg = event.getFormattedMessage(); String requestLogMsg = event.getFormattedMessage();
// desensitization log // desensitization log
return convertMsg(requestLogMsg); return maskSensitiveData(requestLogMsg);
} }
/** public static void addMaskPattern(String maskPattern) {
* deal with sensitive log maskPatterns.add(maskPattern);
*
* @param oriLogMsg original log
*/
private String convertMsg(final String oriLogMsg) {
String tempLogMsg = oriLogMsg;
if (!Strings.isNullOrEmpty(tempLogMsg)) {
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
}
return tempLogMsg;
} }
/** public static String maskSensitiveData(final String logMsg) {
* password regex if (StringUtils.isEmpty(logMsg)) {
* return logMsg;
* @param logMsg original log }
*/ multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE);
static String passwordHandler(Pattern pwdPattern, String logMsg) {
Matcher matcher = pwdPattern.matcher(logMsg);
StringBuffer sb = new StringBuffer(logMsg.length()); StringBuffer sb = new StringBuffer(logMsg.length());
Matcher matcher = multilinePattern.matcher(logMsg);
while (matcher.find()) { while (matcher.find()) {
String password = matcher.group(); String password = matcher.group();
String maskPassword = Strings.repeat(Constants.STAR, password.length()); String maskPassword = Strings.repeat(Constants.STAR, password.length());
matcher.appendReplacement(sb, maskPassword); matcher.appendReplacement(sb, maskPassword);
} }
matcher.appendTail(sb); matcher.appendTail(sb);

27
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java → dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java

@ -15,13 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.log; package org.apache.dolphinscheduler.common.log;
import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler;
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import java.util.regex.Pattern;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -32,11 +26,6 @@ public class SensitiveDataConverterTest {
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\","
+ "\"database\":\"carbond\"," + "\"database\":\"carbond\","
+ "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\","
@ -49,21 +38,17 @@ public class SensitiveDataConverterTest {
+ "\"user\":\"view\"," + "\"user\":\"view\","
+ "\"password\":\"*****\"}"; + "\"password\":\"*****\"}";
@Test
public void convert() {
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
}
/** /**
* mask sensitive logMsg - sql task datasource password * mask sensitive logMsg - sql task datasource password
*/ */
@Test @Test
public void testPwdLogMsgConverter() { public void testPwdLogMsgConverter() {
logger.info("parameter : {}", logMsg); final String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg);
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
logger.info("original parameter : {}", logMsg);
logger.info("masked parameter : {}", maskedLog);
Assertions.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); Assertions.assertEquals(maskLogMsg, maskedLog);
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
} }

2
dolphinscheduler-master/src/main/resources/logback-spring.xml

@ -28,7 +28,7 @@
</appender> </appender>
<conversionRule conversionWord="message" <conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/> converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/> <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator"> <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

2
dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml

@ -48,7 +48,7 @@
<logger name="org.apache.hadoop" level="WARN"/> <logger name="org.apache.hadoop" level="WARN"/>
<conversionRule conversionWord="message" <conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/> converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/> <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator"> <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

1
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java

@ -72,4 +72,5 @@ public final class SqoopConstants {
public static final String UPDATE_KEY = "--update-key"; public static final String UPDATE_KEY = "--update-key";
public static final String UPDATE_MODE = "--update-mode"; public static final String UPDATE_MODE = "--update-mode";
public static final String SQOOP_PASSWORD_REGEX = "(?<=(--password \")).+?(?=\")";
} }

3
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.sqoop; package org.apache.dolphinscheduler.plugin.task.sqoop;
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@ -67,6 +68,8 @@ public class SqoopTask extends AbstractYarnTask {
sqoopTaskExecutionContext = sqoopTaskExecutionContext =
sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
} }
@Override @Override

38
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.sqoop;
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class SqoopTaskTest {
@Test
public void testSqoopPasswordMask() {
final String originalScript =
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile";
final String maskScript =
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile";
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript));
}
}

2
dolphinscheduler-worker/src/main/resources/logback-spring.xml

@ -29,7 +29,7 @@
</appender> </appender>
<conversionRule conversionWord="message" <conversionRule conversionWord="message"
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/> converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/> <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator"> <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

Loading…
Cancel
Save