From 9d10de6efa2b0fa610ae1d53646a38ad58254a45 Mon Sep 17 00:00:00 2001 From: Gallardot Date: Tue, 18 Jul 2023 15:02:20 +0800 Subject: [PATCH] [Improvement][Task] Mask configYaml in task log (#14571) * [Improvement][Task] Mask configYaml in task log Signed-off-by: Gallardot --------- Signed-off-by: Gallardot Co-authored-by: Aaron Wang --- .../common/constants/Constants.java | 5 ++++ .../log/SensitiveDataConverterTest.java | 29 +++++++++++++++++++ .../runner/WorkerTaskExecuteRunnable.java | 3 ++ 3 files changed, 37 insertions(+) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 3bef092488..f214e7aea7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -856,4 +856,9 @@ public final class Constants { */ public static final String DATABASES_QUERY = "show databases"; public static final String DATABASES_QUERY_PG = "SELECT datname FROM pg_database"; + + /** + * K8S sensitive param + */ + public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : \"))).*?(?=(\",\\n))"; } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java index 2268865708..17dcaf6274 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.common.log; +import static org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -62,4 +64,31 @@ public class SensitiveDataConverterTest { Assertions.assertEquals(expectedMsg, maskedLog); } + @Test + public void testK8SLogMsgConverter() { + String msg = "End initialize task {\n" + + " \"taskName\" : \"echo\",\n" + + " \"k8sTaskExecutionContext\" : {\n" + + " \"configYaml\" : \"apiVersion: v1 xxx client-key-data: ==\",\n" + + " \"namespace\" : \"abc\"\n" + + " },\n" + + " \"logBufferEnable\" : false\n" + + "}"; + String maskMsg = "End initialize task {\n" + + " \"taskName\" : \"echo\",\n" + + " \"k8sTaskExecutionContext\" : {\n" + + " \"configYaml\" : \"**************************************\",\n" + + " \"namespace\" : \"abc\"\n" + + " },\n" + + " \"logBufferEnable\" : false\n" + + "}"; + SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); + final String maskedLog = SensitiveDataConverter.maskSensitiveData(msg); + + logger.info("original parameter : {}", msg); + logger.info("masked parameter : {}", maskedLog); + + Assertions.assertEquals(maskMsg, maskedLog); + + } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java index 46629f3634..5e985c628c 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java @@ -19,9 +19,11 @@ package org.apache.dolphinscheduler.server.worker.runner; import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER; import static org.apache.dolphinscheduler.common.constants.Constants.DRY_RUN_FLAG_YES; +import static org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX; import static org.apache.dolphinscheduler.common.constants.Constants.SINGLE_SLASH; import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.log.SensitiveDataConverter; import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; @@ -93,6 +95,7 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; + SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); } protected abstract void executeTask(TaskCallBack taskCallBack);