From 4d5b5b42a42d8d484ebdf81f781fcc6ce317547a Mon Sep 17 00:00:00 2001 From: wind Date: Thu, 13 Jan 2022 22:36:00 +0800 Subject: [PATCH] [cherry-pick-2.0.3] cherry-pick to 2.0.3 (#8008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * finished issue#7527 (#7561) Co-authored-by: 时光 * [Fix-7713] Handling the sensitive data in the log (#7728) * add a feature to handle sensitive data * [cherry-pick-2.0.3]fix-#7740-upgrade_sql #7761 * [cherry-pick-2.0.3][Fix-7203] Remedy the issue about importing a process json file * [cherry-pick-2.0.3][Fix-7857] The user won't be allowed to disable their own account Co-authored-by: 时光 <41109695+yikeshiguang@users.noreply.github.com> Co-authored-by: 时光 Co-authored-by: calvin Co-authored-by: uh001 <96870549+uh001@users.noreply.github.com> --- .../api/aspect/AccessLogAspect.java | 31 ++++++++++++++ .../dolphinscheduler/api/enums/Status.java | 3 +- .../impl/ProcessDefinitionServiceImpl.java | 16 +++++++ .../api/service/impl/UsersServiceImpl.java | 6 +++ .../api/aspect/AccessLogAspectTest.java | 42 +++++++++++++++++++ .../dao/mapper/TaskInstanceMapper.xml | 7 +++- .../mysql/dolphinscheduler_ddl.sql | 10 ++++- .../postgresql/dolphinscheduler_ddl.sql | 6 ++- .../pages/users/_source/createUser.vue | 11 ++++- 9 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java index 14d75c7f74..b20b6d8425 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java @@ -26,7 +26,10 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Set; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.IntStream; import javax.servlet.http.HttpServletRequest; @@ -48,6 +51,10 @@ public class AccessLogAspect { private static final String TRACE_ID = "traceId"; + public static final String sensitiveDataRegEx = "(password=[\'\"]+)(\\S+)([\'\"]+)"; + + private static final Pattern sensitiveDataPattern = Pattern.compile(sensitiveDataRegEx, Pattern.CASE_INSENSITIVE); + @Pointcut("@annotation(org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation)") public void logPointCut(){ // Do nothing because of it's a pointcut @@ -78,6 +85,8 @@ public class AccessLogAspect { // handle args String argsString = parseArgs(proceedingJoinPoint, annotation); + // handle sensitive data in the string + argsString = handleSensitiveData(argsString); logger.info("REQUEST TRACE_ID:{}, LOGIN_USER:{}, URI:{}, METHOD:{}, HANDLER:{}, ARGS:{}", traceId, userName, @@ -119,6 +128,28 @@ public class AccessLogAspect { return argsString; } + protected String handleSensitiveData(String originalData) { + Matcher matcher = sensitiveDataPattern.matcher(originalData.toLowerCase()); + IntStream stream = IntStream.builder().build(); + boolean exists = false; + while (matcher.find()) { + if (matcher.groupCount() == 3) { + stream = IntStream.concat(stream, IntStream.range(matcher.end(1),matcher.end(2))); + exists = true; + } + } + + if (exists) { + char[] chars = originalData.toCharArray(); + stream.forEach(idx -> { + chars[idx] = '*'; + }); + return new String(chars); + } + + return originalData; + } + private String parseLoginInfo(HttpServletRequest request) { String userName = "NOT LOGIN"; User loginUser = (User) (request.getAttribute(Constants.SESSION_USER)); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 824d0d81fa..ac01d651ee 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -348,7 +348,8 @@ public enum Status { VERIFY_ENVIRONMENT_ERROR(1200011, "verify environment error", "验证环境信息错误"), ENVIRONMENT_WORKER_GROUPS_IS_INVALID(1200012, "environment worker groups is invalid format", "环境关联的工作组参数解析错误"), UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR(1200013,"You can't modify the worker group, because the worker group [{0}] and this environment [{1}] already be used in the task [{2}]", - "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中"); + "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中"), + NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT(130020, "Not allow to disable your own account", "不能停用自己的账号"); private final int code; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 9459d0de11..fb9c5038a3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -106,6 +106,7 @@ import org.springframework.web.multipart.MultipartFile; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; @@ -935,6 +936,21 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processTaskRelationLog.setPostTaskVersion(Constants.VERSION_FIRST); taskRelationLogList.add(processTaskRelationLog); } + if (StringUtils.isNotEmpty(processDefinition.getLocations()) && JSONUtils.checkJsonValid(processDefinition.getLocations())) { + ArrayNode arrayNode = JSONUtils.parseArray(processDefinition.getLocations()); + ArrayNode newArrayNode = JSONUtils.createArrayNode(); + for (int i = 0; i < arrayNode.size(); i++) { + ObjectNode newObjectNode = newArrayNode.addObject(); + JsonNode jsonNode = arrayNode.get(i); + Long taskCode = taskCodeMap.get(jsonNode.get("taskCode").asLong()); + if (Objects.nonNull(taskCode)) { + newObjectNode.put("taskCode", taskCode); + newObjectNode.set("x", jsonNode.get("x")); + newObjectNode.set("y", jsonNode.get("y")); + } + } + processDefinition.setLocations(newArrayNode.toString()); + } Map createDagResult = createDagDefine(loginUser, taskRelationLogList, processDefinition, Lists.newArrayList()); if (Status.SUCCESS.equals(createDagResult.get(Constants.STATUS))) { putMsg(createDagResult, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 3b5e0d6d99..4e720820ec 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -407,6 +407,12 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, phone); return result; } + + if (state == 0 && user.getState() != state && loginUser.getId() == user.getId()) { + putMsg(result, Status.NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT); + return result; + } + user.setPhone(phone); user.setQueue(queue); user.setState(state); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java new file mode 100644 index 0000000000..e272e4256e --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.aspect; + +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +/** + * @author Hua Jiang + */ + +public class AccessLogAspectTest { + + private AccessLogAspect accessLogAspect = new AccessLogAspect(); + + @Test + public void testHandleSensitiveData() { + String data = "userPassword='7ad2410b2f4c074479a8937a28a22b8f', email='xxx@qq.com', database='null', userName='root', password='root', other='null'"; + String expected = "userPassword='********************************', email='xxx@qq.com', database='null', userName='root', password='****', other='null'"; + + String actual = accessLogAspect.handleSensitiveData(data); + + Assert.assertEquals(expected, actual); + + } + +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index bdbc5386f3..102b7a4c90 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -81,8 +81,11 @@ #{i} - - and t.start_time > #{startTime} and t.start_time #{endTime} + + and t.start_time ]]> #{startTime} + + + and t.start_time #{endTime} group by t.state diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql index 7e1e605d84..aeaeb4b49e 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql @@ -147,7 +147,10 @@ BEGIN AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='code') THEN - alter table t_ds_project add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `name`; + alter table t_ds_project add `code` bigint(20) COMMENT 'encoding' AFTER `name`; + -- update default value for not null + UPDATE t_ds_project SET code = id; + alter table t_ds_project modify `code` bigint(20) NOT NULL; END IF; END; @@ -399,7 +402,10 @@ alter table t_ds_schedules add timezone_id varchar(40) DEFAULT NULL COMMENT 'tim alter table t_ds_schedules add environment_code bigint(20) DEFAULT '-1' COMMENT 'environment code' AFTER worker_group; -- t_ds_process_definition -alter table t_ds_process_definition add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `id`; +alter table t_ds_process_definition add `code` bigint(20) COMMENT 'encoding' AFTER `id`; +-- update default value for not null +UPDATE t_ds_process_definition SET code = id; +alter table t_ds_process_definition modify `code` bigint(20) NOT NULL; alter table t_ds_process_definition change project_id project_code bigint(20) NOT NULL COMMENT 'project code' AFTER `description`; alter table t_ds_process_definition add `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id' AFTER `locations`; alter table t_ds_process_definition add UNIQUE KEY `process_unique` (`name`,`project_code`) USING BTREE; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql index e5024d9529..deca2d4917 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -74,6 +74,10 @@ BEGIN EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "code" bigint'; EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "warning_group_id" int'; + --update default value for not null + EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_process_definition SET code = id'; + EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_project SET code = id'; + ---drop columns EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_tenant DROP COLUMN IF EXISTS "tenant_name"'; EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_instance DROP COLUMN IF EXISTS "process_instance_json"'; @@ -92,7 +96,7 @@ BEGIN EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "code" SET NOT NULL'; EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "project_code" SET NOT NULL'; EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ADD CONSTRAINT "process_unique" UNIQUE ("name","project_code")'; - EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "description" SET NOT NULL'; + EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_project" ALTER COLUMN "code" SET NOT NULL'; --- drop index EXECUTE 'DROP INDEX IF EXISTS "process_instance_index"'; diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index bb956e9797..25575b6e86 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -97,7 +97,7 @@ - +