[cherry-pick-2.0.3] cherry-pick to 2.0.3 (#8008)

* finished issue#7527 (#7561)

Co-authored-by: 时光 <caizhedong@cai-inc.com>

* [Fix-7713] Handling the sensitive data in the log (#7728)

* add a feature to handle sensitive data

* [cherry-pick-2.0.3]fix-#7740-upgrade_sql #7761

* [cherry-pick-2.0.3][Fix-7203] Remedy the issue about importing a process json file

* [cherry-pick-2.0.3][Fix-7857] The user won't be allowed to disable their own account

Co-authored-by: 时光 <41109695+yikeshiguang@users.noreply.github.com>
Co-authored-by: 时光 <caizhedong@cai-inc.com>
Co-authored-by: calvin <jianghuachinacom@163.com>
Co-authored-by: uh001 <96870549+uh001@users.noreply.github.com>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java

@@ -26,7 +26,10 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Set;
 import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import javax.servlet.http.HttpServletRequest;
 
@@ -48,6 +51,10 @@ public class AccessLogAspect {
 
     private static final String TRACE_ID = "traceId";
 
+    public static final String sensitiveDataRegEx = "(password=[\'\"]+)(\\S+)([\'\"]+)";
+
+    private static final Pattern sensitiveDataPattern = Pattern.compile(sensitiveDataRegEx, Pattern.CASE_INSENSITIVE);
+
     @Pointcut("@annotation(org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation)")
     public void logPointCut(){
         // Do nothing because of it's a pointcut
@@ -78,6 +85,8 @@ public class AccessLogAspect {
 
                 // handle args
                 String argsString = parseArgs(proceedingJoinPoint, annotation);
+                // handle sensitive data in the string
+                argsString = handleSensitiveData(argsString);
                 logger.info("REQUEST TRACE_ID:{}, LOGIN_USER:{}, URI:{}, METHOD:{}, HANDLER:{}, ARGS:{}",
                         traceId,
                         userName,
@@ -119,6 +128,28 @@ public class AccessLogAspect {
         return argsString;
     }
 
+    protected String handleSensitiveData(String originalData) {
+        Matcher matcher = sensitiveDataPattern.matcher(originalData.toLowerCase());
+        IntStream stream = IntStream.builder().build();
+        boolean exists = false;
+        while (matcher.find()) {
+            if (matcher.groupCount() == 3) {
+                stream = IntStream.concat(stream, IntStream.range(matcher.end(1),matcher.end(2)));
+                exists = true;
+            }
+        }
+
+        if (exists) {
+            char[] chars = originalData.toCharArray();
+            stream.forEach(idx -> {
+                chars[idx] = '*';
+            });
+            return new String(chars);
+        }
+
+        return originalData;
+    }
+
     private String parseLoginInfo(HttpServletRequest request) {
         String userName = "NOT LOGIN";
         User loginUser = (User) (request.getAttribute(Constants.SESSION_USER));

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -348,7 +348,8 @@ public enum Status {
     VERIFY_ENVIRONMENT_ERROR(1200011, "verify environment error", "验证环境信息错误"),
     ENVIRONMENT_WORKER_GROUPS_IS_INVALID(1200012, "environment worker groups is invalid format", "环境关联的工作组参数解析错误"),
     UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR(1200013,"You can't modify the worker group, because the worker group [{0}] and this environment [{1}] already be used in the task [{2}]",
-            "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中");
+            "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中"),
+    NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT(130020, "Not allow to disable your own account", "不能停用自己的账号");
 
 
     private final int code;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@@ -106,6 +106,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
@@ -935,6 +936,21 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
             processTaskRelationLog.setPostTaskVersion(Constants.VERSION_FIRST);
             taskRelationLogList.add(processTaskRelationLog);
         }
+        if (StringUtils.isNotEmpty(processDefinition.getLocations()) && JSONUtils.checkJsonValid(processDefinition.getLocations())) {
+            ArrayNode arrayNode = JSONUtils.parseArray(processDefinition.getLocations());
+            ArrayNode newArrayNode = JSONUtils.createArrayNode();
+            for (int i = 0; i < arrayNode.size(); i++) {
+                ObjectNode newObjectNode = newArrayNode.addObject();
+                JsonNode jsonNode = arrayNode.get(i);
+                Long taskCode = taskCodeMap.get(jsonNode.get("taskCode").asLong());
+                if (Objects.nonNull(taskCode)) {
+                    newObjectNode.put("taskCode", taskCode);
+                    newObjectNode.set("x", jsonNode.get("x"));
+                    newObjectNode.set("y", jsonNode.get("y"));
+                }
+            }
+            processDefinition.setLocations(newArrayNode.toString());
+        }
         Map<String, Object> createDagResult = createDagDefine(loginUser, taskRelationLogList, processDefinition, Lists.newArrayList());
         if (Status.SUCCESS.equals(createDagResult.get(Constants.STATUS))) {
             putMsg(createDagResult, Status.SUCCESS);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@@ -407,6 +407,12 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
             putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, phone);
             return result;
         }
+
+        if (state == 0 && user.getState() != state && loginUser.getId() == user.getId()) {
+            putMsg(result, Status.NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT);
+            return result;
+        }
+
         user.setPhone(phone);
         user.setQueue(queue);
         user.setState(state);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.aspect;
+
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+/**
+ * @author Hua Jiang
+ */
+
+public class AccessLogAspectTest {
+
+    private AccessLogAspect accessLogAspect = new AccessLogAspect();
+
+    @Test
+    public void testHandleSensitiveData() {
+        String data = "userPassword='7ad2410b2f4c074479a8937a28a22b8f', email='xxx@qq.com', database='null', userName='root', password='root', other='null'";
+        String expected = "userPassword='********************************', email='xxx@qq.com', database='null', userName='root', password='****', other='null'";
+
+        String actual = accessLogAspect.handleSensitiveData(data);
+
+        Assert.assertEquals(expected, actual);
+
+    }
+
+}

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

@@ -81,8 +81,11 @@
                 #{i}
             </foreach>
         </if>
-        <if test="startTime != null and endTime != null">
-            and t.start_time > #{startTime} and t.start_time <![CDATA[ <= ]]> #{endTime}
+        <if test="startTime != null">
+            and t.start_time <![CDATA[ > ]]> #{startTime}
+        </if>
+        <if test="endTime != null">
+            and t.start_time <![CDATA[ <= ]]> #{endTime}
         </if>
         group by t.state
     </select>

dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql

@@ -147,7 +147,10 @@ BEGIN
                      AND TABLE_SCHEMA=(SELECT DATABASE())
                      AND COLUMN_NAME ='code')
     THEN
-           alter table t_ds_project add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `name`;
+        alter table t_ds_project add `code` bigint(20) COMMENT 'encoding' AFTER `name`;
+        -- update default value for not null
+        UPDATE t_ds_project SET code = id;
+        alter table t_ds_project modify `code` bigint(20) NOT NULL;
     END IF;
 END;
 
@@ -399,7 +402,10 @@ alter table t_ds_schedules add timezone_id varchar(40) DEFAULT NULL COMMENT 'tim
 alter table t_ds_schedules add environment_code bigint(20) DEFAULT '-1' COMMENT 'environment code' AFTER worker_group;
 
 -- t_ds_process_definition
-alter table t_ds_process_definition add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `id`;
+alter table t_ds_process_definition add `code` bigint(20) COMMENT 'encoding' AFTER `id`;
+-- update default value for not null
+UPDATE t_ds_process_definition SET code = id;
+alter table t_ds_process_definition modify `code` bigint(20) NOT NULL;
 alter table t_ds_process_definition change project_id project_code bigint(20) NOT NULL COMMENT 'project code' AFTER `description`;
 alter table t_ds_process_definition add `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id' AFTER `locations`;
 alter table t_ds_process_definition add UNIQUE KEY `process_unique` (`name`,`project_code`) USING BTREE;

dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -74,6 +74,10 @@ BEGIN
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "code" bigint';
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "warning_group_id" int';
 
+    --update default value for not null
+    EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_process_definition SET code = id';
+    EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_project SET code = id';
+
 	---drop columns
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_tenant DROP COLUMN IF EXISTS "tenant_name"';
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_instance DROP COLUMN IF EXISTS "process_instance_json"';
@@ -92,7 +96,7 @@ BEGIN
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "code" SET NOT NULL';
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "project_code" SET NOT NULL';
 	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ADD CONSTRAINT "process_unique" UNIQUE ("name","project_code")';
-	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "description" SET NOT NULL';
+	EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_project" ALTER COLUMN "code" SET NOT NULL';
 
 	--- drop index
 	EXECUTE 'DROP INDEX IF EXISTS "process_instance_index"';

dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue

@@ -97,7 +97,7 @@
             </el-input>
           </template>
         </m-list-box-f>
-        <m-list-box-f style="line-height: 38px;">
+        <m-list-box-f v-if="showState" style="line-height: 38px;">
           <template slot="name">{{$t('State')}}</template>
           <template slot="content">
             <el-radio-group v-model="userState" size="small">
@@ -117,6 +117,7 @@
   import router from '@/conf/home/router'
   import mPopover from '@/module/components/popup/popover'
   import mListBoxF from '@/module/components/listBoxF/listBoxF'
+  import { mapState } from 'vuex'
 
   export default {
     name: 'create-user',
@@ -133,6 +134,7 @@
         phone: '',
         userState: '1',
         tenantList: [],
+        showState: true,
         // Source admin user information
         isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' && router.history.current.name !== 'account'
       }
@@ -292,6 +294,7 @@
     watch: {},
     created () {
       // Administrator gets tenant list
+      this.showState = true
       if (this.isADMIN) {
         Promise.all([this._getQueueList(), this._getTenantList()]).then(() => {
           if (this.item) {
@@ -301,6 +304,7 @@
             this.phone = this.item.phone
             this.state = this.item.state
             this.userState = this.item.state + '' || '1'
+            this.showState = this.item.id !== this.userInfo.id
             if (this.fromUserInfo || this.item.tenantId) {
               this.tenantId = this.item.tenantId
             }
@@ -320,6 +324,7 @@
           this.phone = this.item.phone
           this.state = this.item.state
           this.userState = this.state + '' || '1'
+          this.showState = this.item.id !== this.userInfo.id
           if (this.fromUserInfo || this.item.tenantId) {
             this.tenantId = this.item.tenantId
           }
@@ -335,7 +340,9 @@
       }
     },
     mounted () {
-
+    },
+    computed: {
+      ...mapState('user', ['userInfo'])
     },
     components: { mPopover, mListBoxF }
   }